2024-11-23 15:40:03,049 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-23 15:40:03,063 main DEBUG Took 0.012303 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-23 15:40:03,063 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-23 15:40:03,064 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-23 15:40:03,065 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-23 15:40:03,066 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:40:03,075 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-23 15:40:03,087 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:40:03,088 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:40:03,089 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:40:03,089 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:40:03,089 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:40:03,090 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:40:03,091 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:40:03,091 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:40:03,091 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:40:03,092 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:40:03,092 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:40:03,093 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:40:03,093 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:40:03,093 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:40:03,094 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:40:03,094 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:40:03,094 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:40:03,095 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:40:03,095 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:40:03,095 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:40:03,096 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:40:03,096 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:40:03,097 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:40:03,097 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 15:40:03,097 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:40:03,097 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-23 15:40:03,099 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 15:40:03,100 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-23 15:40:03,102 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-23 15:40:03,102 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-23 15:40:03,103 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-23 15:40:03,104 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-23 15:40:03,111 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-23 15:40:03,113 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-23 15:40:03,115 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-23 15:40:03,115 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-23 15:40:03,115 main DEBUG createAppenders(={Console}) 2024-11-23 15:40:03,116 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-23 15:40:03,116 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-23 15:40:03,116 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-23 15:40:03,117 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-23 15:40:03,117 main DEBUG OutputStream closed 2024-11-23 15:40:03,117 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-23 15:40:03,118 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-23 15:40:03,118 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-23 15:40:03,188 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-23 15:40:03,190 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-23 15:40:03,191 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-23 15:40:03,192 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-23 15:40:03,193 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-23 15:40:03,193 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-23 15:40:03,194 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-23 15:40:03,194 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-23 15:40:03,194 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-23 15:40:03,194 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-23 15:40:03,195 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-23 15:40:03,195 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-23 15:40:03,195 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-23 15:40:03,196 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-23 15:40:03,196 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-23 15:40:03,196 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-23 15:40:03,196 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-23 15:40:03,197 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-23 15:40:03,199 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23 15:40:03,200 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-23 15:40:03,200 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-23 15:40:03,200 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-23T15:40:03,473 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d 2024-11-23 15:40:03,476 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-23 15:40:03,477 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23T15:40:03,485 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-23T15:40:03,509 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-23T15:40:03,524 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/cluster_bcb5f073-d146-1328-644a-da8d77accbc4, deleteOnExit=true 2024-11-23T15:40:03,525 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-23T15:40:03,526 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/test.cache.data in system properties and HBase conf 2024-11-23T15:40:03,526 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/hadoop.tmp.dir in system properties and HBase conf 2024-11-23T15:40:03,527 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/hadoop.log.dir in system properties and HBase conf 2024-11-23T15:40:03,528 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-23T15:40:03,528 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-23T15:40:03,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-23T15:40:03,621 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-23T15:40:03,722 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T15:40:03,726 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-23T15:40:03,727 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-23T15:40:03,728 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-23T15:40:03,728 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T15:40:03,729 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-23T15:40:03,729 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-23T15:40:03,730 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T15:40:03,730 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T15:40:03,731 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-23T15:40:03,731 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/nfs.dump.dir in system properties and HBase conf 2024-11-23T15:40:03,731 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/java.io.tmpdir in system properties and HBase conf 2024-11-23T15:40:03,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T15:40:03,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-23T15:40:03,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-23T15:40:04,887 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-23T15:40:04,997 INFO [Time-limited test {}] log.Log(170): Logging initialized @2642ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-23T15:40:05,108 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T15:40:05,191 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T15:40:05,213 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T15:40:05,213 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T15:40:05,215 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T15:40:05,229 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T15:40:05,232 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/hadoop.log.dir/,AVAILABLE} 2024-11-23T15:40:05,233 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T15:40:05,527 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/java.io.tmpdir/jetty-localhost-45201-hadoop-hdfs-3_4_1-tests_jar-_-any-7603670001973190992/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T15:40:05,537 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:45201} 2024-11-23T15:40:05,537 INFO [Time-limited test {}] server.Server(415): Started @3184ms 2024-11-23T15:40:06,122 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T15:40:06,132 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T15:40:06,135 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T15:40:06,136 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T15:40:06,136 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T15:40:06,139 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/hadoop.log.dir/,AVAILABLE} 2024-11-23T15:40:06,139 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T15:40:06,252 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/java.io.tmpdir/jetty-localhost-43513-hadoop-hdfs-3_4_1-tests_jar-_-any-8650972859540639093/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T15:40:06,253 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:43513} 2024-11-23T15:40:06,253 INFO [Time-limited test {}] server.Server(415): Started @3900ms 2024-11-23T15:40:06,306 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T15:40:07,291 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/cluster_bcb5f073-d146-1328-644a-da8d77accbc4/data/data1/current/BP-1051640608-172.17.0.2-1732376404362/current, will proceed with Du for space computation calculation, 2024-11-23T15:40:07,291 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/cluster_bcb5f073-d146-1328-644a-da8d77accbc4/data/data2/current/BP-1051640608-172.17.0.2-1732376404362/current, will proceed with Du for space computation calculation, 2024-11-23T15:40:07,327 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T15:40:07,373 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xce0c78cfec536f1f with lease ID 0xe588400aedf9583b: Processing first storage report for DS-97e4ccb9-2c12-454c-9d89-96c85eb980a8 from datanode DatanodeRegistration(127.0.0.1:35405, datanodeUuid=637a375a-7b58-4664-9b52-e85f966970ab, infoPort=44017, infoSecurePort=0, ipcPort=36743, storageInfo=lv=-57;cid=testClusterID;nsid=1694623711;c=1732376404362) 2024-11-23T15:40:07,375 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xce0c78cfec536f1f with lease ID 0xe588400aedf9583b: from storage DS-97e4ccb9-2c12-454c-9d89-96c85eb980a8 node DatanodeRegistration(127.0.0.1:35405, datanodeUuid=637a375a-7b58-4664-9b52-e85f966970ab, infoPort=44017, infoSecurePort=0, ipcPort=36743, storageInfo=lv=-57;cid=testClusterID;nsid=1694623711;c=1732376404362), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-23T15:40:07,375 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xce0c78cfec536f1f with lease ID 0xe588400aedf9583b: Processing first storage report for DS-923f6e11-d6b3-41a0-92ab-d2c161e8dc1c from datanode DatanodeRegistration(127.0.0.1:35405, datanodeUuid=637a375a-7b58-4664-9b52-e85f966970ab, infoPort=44017, infoSecurePort=0, ipcPort=36743, storageInfo=lv=-57;cid=testClusterID;nsid=1694623711;c=1732376404362) 2024-11-23T15:40:07,375 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xce0c78cfec536f1f with lease ID 0xe588400aedf9583b: from storage DS-923f6e11-d6b3-41a0-92ab-d2c161e8dc1c node DatanodeRegistration(127.0.0.1:35405, datanodeUuid=637a375a-7b58-4664-9b52-e85f966970ab, infoPort=44017, infoSecurePort=0, ipcPort=36743, storageInfo=lv=-57;cid=testClusterID;nsid=1694623711;c=1732376404362), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T15:40:07,454 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d 2024-11-23T15:40:07,539 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/cluster_bcb5f073-d146-1328-644a-da8d77accbc4/zookeeper_0, clientPort=58385, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/cluster_bcb5f073-d146-1328-644a-da8d77accbc4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/cluster_bcb5f073-d146-1328-644a-da8d77accbc4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T15:40:07,548 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58385 2024-11-23T15:40:07,560 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:40:07,564 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:40:07,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741825_1001 (size=7) 2024-11-23T15:40:08,180 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e with version=8 2024-11-23T15:40:08,180 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/hbase-staging 2024-11-23T15:40:08,286 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-23T15:40:08,569 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b712f9af2c12:0 server-side Connection retries=45 2024-11-23T15:40:08,580 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T15:40:08,581 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T15:40:08,586 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T15:40:08,587 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T15:40:08,587 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T15:40:08,720 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T15:40:08,772 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-23T15:40:08,780 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-23T15:40:08,784 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T15:40:08,807 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2495 (auto-detected) 2024-11-23T15:40:08,808 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-23T15:40:08,829 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42351 2024-11-23T15:40:08,852 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42351 connecting to ZooKeeper ensemble=127.0.0.1:58385 2024-11-23T15:40:08,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:423510x0, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T15:40:08,940 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42351-0x10169a3c09b0000 connected 2024-11-23T15:40:09,036 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:40:09,039 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:40:09,048 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T15:40:09,052 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e, hbase.cluster.distributed=false 2024-11-23T15:40:09,077 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T15:40:09,087 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42351 2024-11-23T15:40:09,088 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42351 2024-11-23T15:40:09,088 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42351 2024-11-23T15:40:09,095 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42351 2024-11-23T15:40:09,095 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42351 2024-11-23T15:40:09,244 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b712f9af2c12:0 server-side Connection retries=45 2024-11-23T15:40:09,247 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T15:40:09,247 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T15:40:09,247 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T15:40:09,247 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T15:40:09,248 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T15:40:09,250 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T15:40:09,253 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T15:40:09,254 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46589 2024-11-23T15:40:09,256 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46589 connecting to ZooKeeper ensemble=127.0.0.1:58385 2024-11-23T15:40:09,257 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:40:09,260 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:40:09,285 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:465890x0, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T15:40:09,286 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46589-0x10169a3c09b0001 connected 2024-11-23T15:40:09,286 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:465890x0, quorum=127.0.0.1:58385, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T15:40:09,291 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T15:40:09,303 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T15:40:09,306 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46589-0x10169a3c09b0001, quorum=127.0.0.1:58385, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T15:40:09,313 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46589-0x10169a3c09b0001, quorum=127.0.0.1:58385, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T15:40:09,316 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46589 2024-11-23T15:40:09,317 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46589 2024-11-23T15:40:09,321 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46589 2024-11-23T15:40:09,322 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46589 2024-11-23T15:40:09,322 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46589 2024-11-23T15:40:09,345 DEBUG [M:0;b712f9af2c12:42351 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b712f9af2c12:42351 2024-11-23T15:40:09,347 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b712f9af2c12,42351,1732376408377 2024-11-23T15:40:09,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46589-0x10169a3c09b0001, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T15:40:09,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T15:40:09,371 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b712f9af2c12,42351,1732376408377 2024-11-23T15:40:09,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46589-0x10169a3c09b0001, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T15:40:09,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:40:09,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46589-0x10169a3c09b0001, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:40:09,403 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T15:40:09,404 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b712f9af2c12,42351,1732376408377 from backup master directory 2024-11-23T15:40:09,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46589-0x10169a3c09b0001, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T15:40:09,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b712f9af2c12,42351,1732376408377 2024-11-23T15:40:09,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T15:40:09,417 WARN [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T15:40:09,417 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b712f9af2c12,42351,1732376408377 2024-11-23T15:40:09,420 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-23T15:40:09,421 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-23T15:40:09,478 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/hbase.id] with ID: f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193 2024-11-23T15:40:09,478 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/.tmp/hbase.id 2024-11-23T15:40:09,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741826_1002 (size=42) 2024-11-23T15:40:09,891 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/.tmp/hbase.id]:[hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/hbase.id] 2024-11-23T15:40:09,937 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T15:40:09,941 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T15:40:09,961 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-23T15:40:09,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46589-0x10169a3c09b0001, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:40:09,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:40:09,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741827_1003 (size=196) 2024-11-23T15:40:10,410 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:40:10,413 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T15:40:10,430 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:10,435 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:40:10,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741828_1004 (size=1189) 2024-11-23T15:40:10,889 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/data/master/store 2024-11-23T15:40:10,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741829_1005 (size=34) 2024-11-23T15:40:11,324 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-23T15:40:11,328 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:40:11,330 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T15:40:11,330 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:40:11,330 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:40:11,332 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T15:40:11,332 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:40:11,332 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:40:11,334 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732376411329Disabling compacts and flushes for region at 1732376411329Disabling writes for close at 1732376411332 (+3 ms)Writing region close event to WAL at 1732376411332Closed at 1732376411332 2024-11-23T15:40:11,336 WARN [master/b712f9af2c12:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/data/master/store/.initializing 2024-11-23T15:40:11,336 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/WALs/b712f9af2c12,42351,1732376408377 2024-11-23T15:40:11,344 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T15:40:11,360 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b712f9af2c12%2C42351%2C1732376408377, suffix=, logDir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/WALs/b712f9af2c12,42351,1732376408377, archiveDir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/oldWALs, maxLogs=10 2024-11-23T15:40:11,384 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/WALs/b712f9af2c12,42351,1732376408377/b712f9af2c12%2C42351%2C1732376408377.1732376411365, exclude list is [], retry=0 2024-11-23T15:40:11,402 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35405,DS-97e4ccb9-2c12-454c-9d89-96c85eb980a8,DISK] 2024-11-23T15:40:11,406 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-23T15:40:11,442 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/WALs/b712f9af2c12,42351,1732376408377/b712f9af2c12%2C42351%2C1732376408377.1732376411365 2024-11-23T15:40:11,443 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44017:44017)] 2024-11-23T15:40:11,443 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:40:11,444 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:40:11,447 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:40:11,447 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:40:11,480 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:40:11,508 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T15:40:11,511 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:11,513 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:40:11,514 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:40:11,517 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T15:40:11,517 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:11,518 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:40:11,518 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:40:11,521 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T15:40:11,521 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:11,522 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:40:11,523 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:40:11,525 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T15:40:11,525 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:11,526 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:40:11,527 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:40:11,530 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:40:11,532 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:40:11,537 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:40:11,538 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:40:11,542 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T15:40:11,545 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T15:40:11,550 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:40:11,551 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65891421, jitterRate=-0.018141314387321472}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T15:40:11,557 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732376411458Initializing all the Stores at 1732376411460 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732376411461 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376411461Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376411461Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376411462 (+1 ms)Cleaning up temporary data from old regions at 1732376411538 (+76 ms)Region opened successfully at 1732376411557 (+19 ms) 2024-11-23T15:40:11,559 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T15:40:11,589 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d03a7d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b712f9af2c12/172.17.0.2:0 2024-11-23T15:40:11,617 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T15:40:11,627 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T15:40:11,627 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T15:40:11,630 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T15:40:11,631 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-23T15:40:11,636 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-23T15:40:11,636 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T15:40:11,659 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T15:40:11,668 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T15:40:11,690 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T15:40:11,693 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T15:40:11,697 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T15:40:11,709 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T15:40:11,712 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T15:40:11,716 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T15:40:11,725 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T15:40:11,728 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T15:40:11,740 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T15:40:11,761 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T15:40:11,767 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T15:40:11,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T15:40:11,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46589-0x10169a3c09b0001, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T15:40:11,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:40:11,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46589-0x10169a3c09b0001, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:40:11,780 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b712f9af2c12,42351,1732376408377, sessionid=0x10169a3c09b0000, setting cluster-up flag (Was=false) 2024-11-23T15:40:11,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46589-0x10169a3c09b0001, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:40:11,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:40:11,834 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T15:40:11,837 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b712f9af2c12,42351,1732376408377 2024-11-23T15:40:11,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:40:11,859 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46589-0x10169a3c09b0001, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:40:11,884 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T15:40:11,890 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b712f9af2c12,42351,1732376408377 2024-11-23T15:40:11,898 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T15:40:11,931 INFO [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(746): ClusterId : f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193 2024-11-23T15:40:11,934 DEBUG [RS:0;b712f9af2c12:46589 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T15:40:11,971 DEBUG [RS:0;b712f9af2c12:46589 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T15:40:11,971 DEBUG [RS:0;b712f9af2c12:46589 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T15:40:11,978 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T15:40:11,984 DEBUG [RS:0;b712f9af2c12:46589 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T15:40:11,984 DEBUG [RS:0;b712f9af2c12:46589 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54728625, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b712f9af2c12/172.17.0.2:0 2024-11-23T15:40:11,989 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T15:40:11,996 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T15:40:12,001 DEBUG [RS:0;b712f9af2c12:46589 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b712f9af2c12:46589 2024-11-23T15:40:12,005 INFO [RS:0;b712f9af2c12:46589 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T15:40:12,005 INFO [RS:0;b712f9af2c12:46589 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T15:40:12,006 DEBUG [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T15:40:12,002 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b712f9af2c12,42351,1732376408377 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T15:40:12,009 INFO [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(2659): reportForDuty to master=b712f9af2c12,42351,1732376408377 with port=46589, startcode=1732376409194 2024-11-23T15:40:12,011 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b712f9af2c12:0, corePoolSize=5, maxPoolSize=5 2024-11-23T15:40:12,011 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b712f9af2c12:0, corePoolSize=5, maxPoolSize=5 2024-11-23T15:40:12,012 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b712f9af2c12:0, corePoolSize=5, maxPoolSize=5 2024-11-23T15:40:12,012 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b712f9af2c12:0, corePoolSize=5, maxPoolSize=5 2024-11-23T15:40:12,012 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b712f9af2c12:0, corePoolSize=10, maxPoolSize=10 2024-11-23T15:40:12,013 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:40:12,013 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b712f9af2c12:0, corePoolSize=2, maxPoolSize=2 2024-11-23T15:40:12,013 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:40:12,015 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732376442015 2024-11-23T15:40:12,017 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T15:40:12,019 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T15:40:12,019 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T15:40:12,020 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T15:40:12,020 DEBUG [RS:0;b712f9af2c12:46589 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T15:40:12,022 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T15:40:12,023 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T15:40:12,023 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T15:40:12,023 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T15:40:12,024 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T15:40:12,027 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:12,027 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T15:40:12,027 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T15:40:12,028 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T15:40:12,029 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T15:40:12,031 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T15:40:12,031 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T15:40:12,035 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b712f9af2c12:0:becomeActiveMaster-HFileCleaner.large.0-1732376412032,5,FailOnTimeoutGroup] 2024-11-23T15:40:12,035 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b712f9af2c12:0:becomeActiveMaster-HFileCleaner.small.0-1732376412035,5,FailOnTimeoutGroup] 2024-11-23T15:40:12,036 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T15:40:12,036 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T15:40:12,037 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T15:40:12,038 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T15:40:12,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741831_1007 (size=1321) 2024-11-23T15:40:12,059 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51399, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T15:40:12,065 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42351 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b712f9af2c12,46589,1732376409194 2024-11-23T15:40:12,066 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42351 {}] master.ServerManager(517): Registering regionserver=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:12,079 DEBUG [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e 2024-11-23T15:40:12,079 DEBUG [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46341 2024-11-23T15:40:12,079 DEBUG [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T15:40:12,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T15:40:12,099 DEBUG [RS:0;b712f9af2c12:46589 {}] zookeeper.ZKUtil(111): regionserver:46589-0x10169a3c09b0001, quorum=127.0.0.1:58385, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b712f9af2c12,46589,1732376409194 2024-11-23T15:40:12,099 WARN [RS:0;b712f9af2c12:46589 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T15:40:12,100 INFO [RS:0;b712f9af2c12:46589 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:40:12,100 DEBUG [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/WALs/b712f9af2c12,46589,1732376409194 2024-11-23T15:40:12,102 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b712f9af2c12,46589,1732376409194] 2024-11-23T15:40:12,123 INFO [RS:0;b712f9af2c12:46589 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T15:40:12,138 INFO [RS:0;b712f9af2c12:46589 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T15:40:12,143 INFO [RS:0;b712f9af2c12:46589 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T15:40:12,143 INFO [RS:0;b712f9af2c12:46589 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T15:40:12,144 INFO [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T15:40:12,148 INFO [RS:0;b712f9af2c12:46589 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T15:40:12,149 INFO [RS:0;b712f9af2c12:46589 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T15:40:12,150 DEBUG [RS:0;b712f9af2c12:46589 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:40:12,150 DEBUG [RS:0;b712f9af2c12:46589 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:40:12,150 DEBUG [RS:0;b712f9af2c12:46589 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:40:12,150 DEBUG [RS:0;b712f9af2c12:46589 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:40:12,150 DEBUG [RS:0;b712f9af2c12:46589 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:40:12,151 DEBUG [RS:0;b712f9af2c12:46589 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b712f9af2c12:0, corePoolSize=2, maxPoolSize=2 2024-11-23T15:40:12,151 DEBUG [RS:0;b712f9af2c12:46589 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:40:12,151 DEBUG [RS:0;b712f9af2c12:46589 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:40:12,151 DEBUG [RS:0;b712f9af2c12:46589 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:40:12,151 DEBUG [RS:0;b712f9af2c12:46589 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:40:12,151 DEBUG [RS:0;b712f9af2c12:46589 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:40:12,151 DEBUG [RS:0;b712f9af2c12:46589 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b712f9af2c12:0, corePoolSize=1, maxPoolSize=1 2024-11-23T15:40:12,152 DEBUG [RS:0;b712f9af2c12:46589 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b712f9af2c12:0, corePoolSize=3, maxPoolSize=3 2024-11-23T15:40:12,152 DEBUG [RS:0;b712f9af2c12:46589 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0, corePoolSize=3, maxPoolSize=3 2024-11-23T15:40:12,153 INFO [RS:0;b712f9af2c12:46589 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T15:40:12,153 INFO [RS:0;b712f9af2c12:46589 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T15:40:12,153 INFO [RS:0;b712f9af2c12:46589 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T15:40:12,153 INFO [RS:0;b712f9af2c12:46589 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T15:40:12,153 INFO [RS:0;b712f9af2c12:46589 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T15:40:12,153 INFO [RS:0;b712f9af2c12:46589 {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,46589,1732376409194-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T15:40:12,174 INFO [RS:0;b712f9af2c12:46589 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T15:40:12,175 INFO [RS:0;b712f9af2c12:46589 {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,46589,1732376409194-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T15:40:12,176 INFO [RS:0;b712f9af2c12:46589 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T15:40:12,176 INFO [RS:0;b712f9af2c12:46589 {}] regionserver.Replication(171): b712f9af2c12,46589,1732376409194 started 2024-11-23T15:40:12,193 INFO [RS:0;b712f9af2c12:46589 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T15:40:12,194 INFO [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(1482): Serving as b712f9af2c12,46589,1732376409194, RpcServer on b712f9af2c12/172.17.0.2:46589, sessionid=0x10169a3c09b0001 2024-11-23T15:40:12,195 DEBUG [RS:0;b712f9af2c12:46589 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T15:40:12,195 DEBUG [RS:0;b712f9af2c12:46589 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b712f9af2c12,46589,1732376409194 2024-11-23T15:40:12,195 DEBUG [RS:0;b712f9af2c12:46589 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b712f9af2c12,46589,1732376409194' 2024-11-23T15:40:12,195 DEBUG [RS:0;b712f9af2c12:46589 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T15:40:12,196 DEBUG [RS:0;b712f9af2c12:46589 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T15:40:12,197 DEBUG [RS:0;b712f9af2c12:46589 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T15:40:12,197 DEBUG [RS:0;b712f9af2c12:46589 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T15:40:12,197 DEBUG [RS:0;b712f9af2c12:46589 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b712f9af2c12,46589,1732376409194 2024-11-23T15:40:12,197 DEBUG [RS:0;b712f9af2c12:46589 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b712f9af2c12,46589,1732376409194' 2024-11-23T15:40:12,197 DEBUG [RS:0;b712f9af2c12:46589 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T15:40:12,198 DEBUG [RS:0;b712f9af2c12:46589 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T15:40:12,199 DEBUG [RS:0;b712f9af2c12:46589 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T15:40:12,199 INFO [RS:0;b712f9af2c12:46589 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T15:40:12,199 INFO [RS:0;b712f9af2c12:46589 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T15:40:12,307 INFO [RS:0;b712f9af2c12:46589 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T15:40:12,311 INFO [RS:0;b712f9af2c12:46589 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b712f9af2c12%2C46589%2C1732376409194, suffix=, logDir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/WALs/b712f9af2c12,46589,1732376409194, archiveDir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/oldWALs, maxLogs=32 2024-11-23T15:40:12,325 DEBUG [RS:0;b712f9af2c12:46589 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/WALs/b712f9af2c12,46589,1732376409194/b712f9af2c12%2C46589%2C1732376409194.1732376412313, exclude list is [], retry=0 2024-11-23T15:40:12,329 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35405,DS-97e4ccb9-2c12-454c-9d89-96c85eb980a8,DISK] 2024-11-23T15:40:12,332 INFO [RS:0;b712f9af2c12:46589 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/WALs/b712f9af2c12,46589,1732376409194/b712f9af2c12%2C46589%2C1732376409194.1732376412313 2024-11-23T15:40:12,333 DEBUG [RS:0;b712f9af2c12:46589 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44017:44017)] 2024-11-23T15:40:12,445 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T15:40:12,446 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e 2024-11-23T15:40:12,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741833_1009 (size=32) 2024-11-23T15:40:12,859 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:40:12,862 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T15:40:12,865 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T15:40:12,865 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:12,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:40:12,867 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T15:40:12,869 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T15:40:12,869 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:12,871 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:40:12,871 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T15:40:12,874 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T15:40:12,874 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:12,876 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:40:12,876 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T15:40:12,878 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T15:40:12,879 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:12,880 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:40:12,880 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T15:40:12,882 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740 2024-11-23T15:40:12,882 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740 2024-11-23T15:40:12,887 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T15:40:12,887 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T15:40:12,889 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T15:40:12,892 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T15:40:12,899 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:40:12,900 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66260753, jitterRate=-0.01263783872127533}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T15:40:12,903 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732376412860Initializing all the Stores at 1732376412862 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732376412862Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732376412862Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376412862Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732376412862Cleaning up temporary data from old regions at 1732376412888 (+26 ms)Region opened successfully at 1732376412903 (+15 ms) 2024-11-23T15:40:12,903 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T15:40:12,903 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T15:40:12,903 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T15:40:12,903 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T15:40:12,903 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T15:40:12,905 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T15:40:12,906 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732376412903Disabling compacts and flushes for region at 1732376412903Disabling writes for close at 1732376412903Writing region close event to WAL at 1732376412905 (+2 ms)Closed at 1732376412905 2024-11-23T15:40:12,909 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T15:40:12,909 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T15:40:12,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T15:40:12,923 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T15:40:12,926 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T15:40:13,079 DEBUG [b712f9af2c12:42351 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-23T15:40:13,090 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:13,096 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b712f9af2c12,46589,1732376409194, state=OPENING 2024-11-23T15:40:13,150 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T15:40:13,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:40:13,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46589-0x10169a3c09b0001, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:40:13,159 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T15:40:13,159 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T15:40:13,161 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T15:40:13,164 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b712f9af2c12,46589,1732376409194}] 2024-11-23T15:40:13,344 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T15:40:13,347 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43063, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T15:40:13,358 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T15:40:13,359 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T15:40:13,359 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T15:40:13,362 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b712f9af2c12%2C46589%2C1732376409194.meta, suffix=.meta, logDir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/WALs/b712f9af2c12,46589,1732376409194, archiveDir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/oldWALs, maxLogs=32 2024-11-23T15:40:13,378 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/WALs/b712f9af2c12,46589,1732376409194/b712f9af2c12%2C46589%2C1732376409194.meta.1732376413364.meta, exclude list is [], retry=0 2024-11-23T15:40:13,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35405,DS-97e4ccb9-2c12-454c-9d89-96c85eb980a8,DISK] 2024-11-23T15:40:13,385 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/WALs/b712f9af2c12,46589,1732376409194/b712f9af2c12%2C46589%2C1732376409194.meta.1732376413364.meta 2024-11-23T15:40:13,386 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44017:44017)] 2024-11-23T15:40:13,386 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:40:13,388 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T15:40:13,391 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T15:40:13,396 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T15:40:13,400 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T15:40:13,401 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:40:13,401 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T15:40:13,402 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T15:40:13,405 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T15:40:13,407 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T15:40:13,407 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:13,408 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:40:13,408 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T15:40:13,410 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T15:40:13,410 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:13,410 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:40:13,411 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T15:40:13,412 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T15:40:13,412 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:13,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:40:13,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T15:40:13,415 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T15:40:13,415 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:13,416 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T15:40:13,416 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T15:40:13,418 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740 2024-11-23T15:40:13,420 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740 2024-11-23T15:40:13,422 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T15:40:13,422 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T15:40:13,423 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T15:40:13,426 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T15:40:13,427 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68734256, jitterRate=0.02422022819519043}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T15:40:13,428 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T15:40:13,429 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732376413402Writing region info on filesystem at 1732376413402Initializing all the Stores at 1732376413404 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732376413404Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732376413405 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376413405Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732376413405Cleaning up temporary data from old regions at 1732376413422 (+17 ms)Running coprocessor post-open hooks at 1732376413428 (+6 ms)Region opened successfully at 1732376413429 (+1 ms) 2024-11-23T15:40:13,436 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732376413335 2024-11-23T15:40:13,446 DEBUG [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T15:40:13,447 INFO [RS_OPEN_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T15:40:13,449 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:13,452 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b712f9af2c12,46589,1732376409194, state=OPEN 2024-11-23T15:40:13,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T15:40:13,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46589-0x10169a3c09b0001, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T15:40:13,501 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T15:40:13,501 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T15:40:13,502 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:13,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T15:40:13,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b712f9af2c12,46589,1732376409194 in 339 msec 2024-11-23T15:40:13,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T15:40:13,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 594 msec 2024-11-23T15:40:13,518 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T15:40:13,518 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T15:40:13,543 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:40:13,545 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:40:13,569 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:40:13,573 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54283, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:40:13,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6690 sec 2024-11-23T15:40:13,605 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732376413604, completionTime=-1 2024-11-23T15:40:13,608 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-23T15:40:13,609 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T15:40:13,646 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-23T15:40:13,646 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732376473646 2024-11-23T15:40:13,647 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732376533647 2024-11-23T15:40:13,647 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 38 msec 2024-11-23T15:40:13,650 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,42351,1732376408377-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T15:40:13,650 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,42351,1732376408377-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T15:40:13,650 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,42351,1732376408377-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T15:40:13,652 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b712f9af2c12:42351, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T15:40:13,653 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T15:40:13,654 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T15:40:13,660 DEBUG [master/b712f9af2c12:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T15:40:13,682 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.264sec 2024-11-23T15:40:13,684 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T15:40:13,686 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T15:40:13,687 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T15:40:13,687 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T15:40:13,687 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T15:40:13,688 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,42351,1732376408377-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T15:40:13,689 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,42351,1732376408377-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T15:40:13,697 DEBUG [master/b712f9af2c12:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T15:40:13,697 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T15:40:13,698 INFO [master/b712f9af2c12:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b712f9af2c12,42351,1732376408377-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T15:40:13,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:13,743 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-23T15:40:13,743 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-23T15:40:13,746 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:40:13,749 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:40:13,756 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:40:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:40:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:40:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d86f895, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:40:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:40:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:13,767 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:40:13,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:13,770 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:40:13,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:40:13,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:40:13,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:40:13,800 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b712f9af2c12,42351,1732376408377 2024-11-23T15:40:13,812 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=322, ProcessCount=11, AvailableMemoryMB=8520 2024-11-23T15:40:13,822 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T15:40:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is b712f9af2c12,42351,1732376408377 2024-11-23T15:40:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@65d48f8e 2024-11-23T15:40:13,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T15:40:13,847 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42248, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T15:40:13,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T15:40:13,855 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:40:13,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T15:40:13,870 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T15:40:13,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-23T15:40:13,873 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:13,875 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T15:40:13,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T15:40:13,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741835_1011 (size=959) 2024-11-23T15:40:13,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T15:40:14,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T15:40:14,294 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e 2024-11-23T15:40:14,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741836_1012 (size=53) 2024-11-23T15:40:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T15:40:14,707 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:40:14,708 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing dad1202300d1ddd3a95c32af1ad98ed0, disabling compactions & flushes 2024-11-23T15:40:14,708 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:14,708 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:14,708 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. after waiting 0 ms 2024-11-23T15:40:14,708 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:14,708 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:14,708 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for dad1202300d1ddd3a95c32af1ad98ed0: Waiting for close lock at 1732376414708Disabling compacts and flushes for region at 1732376414708Disabling writes for close at 1732376414708Writing region close event to WAL at 1732376414708Closed at 1732376414708 2024-11-23T15:40:14,711 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T15:40:14,716 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732376414711"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732376414711"}]},"ts":"1732376414711"} 2024-11-23T15:40:14,721 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T15:40:14,723 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T15:40:14,726 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376414723"}]},"ts":"1732376414723"} 2024-11-23T15:40:14,730 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T15:40:14,732 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dad1202300d1ddd3a95c32af1ad98ed0, ASSIGN}] 2024-11-23T15:40:14,735 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dad1202300d1ddd3a95c32af1ad98ed0, ASSIGN 2024-11-23T15:40:14,737 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dad1202300d1ddd3a95c32af1ad98ed0, ASSIGN; state=OFFLINE, location=b712f9af2c12,46589,1732376409194; forceNewPlan=false, retain=false 2024-11-23T15:40:14,888 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dad1202300d1ddd3a95c32af1ad98ed0, regionState=OPENING, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:14,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dad1202300d1ddd3a95c32af1ad98ed0, ASSIGN because future has completed 2024-11-23T15:40:14,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194}] 2024-11-23T15:40:15,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T15:40:15,062 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:15,062 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:40:15,063 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:15,063 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:40:15,063 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:15,063 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:15,066 INFO [StoreOpener-dad1202300d1ddd3a95c32af1ad98ed0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:15,068 INFO [StoreOpener-dad1202300d1ddd3a95c32af1ad98ed0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dad1202300d1ddd3a95c32af1ad98ed0 columnFamilyName A 2024-11-23T15:40:15,068 DEBUG [StoreOpener-dad1202300d1ddd3a95c32af1ad98ed0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:15,069 INFO [StoreOpener-dad1202300d1ddd3a95c32af1ad98ed0-1 {}] regionserver.HStore(327): Store=dad1202300d1ddd3a95c32af1ad98ed0/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:40:15,070 INFO [StoreOpener-dad1202300d1ddd3a95c32af1ad98ed0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:15,072 INFO [StoreOpener-dad1202300d1ddd3a95c32af1ad98ed0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dad1202300d1ddd3a95c32af1ad98ed0 columnFamilyName B 2024-11-23T15:40:15,072 DEBUG [StoreOpener-dad1202300d1ddd3a95c32af1ad98ed0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:15,073 INFO [StoreOpener-dad1202300d1ddd3a95c32af1ad98ed0-1 {}] regionserver.HStore(327): Store=dad1202300d1ddd3a95c32af1ad98ed0/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:40:15,073 INFO [StoreOpener-dad1202300d1ddd3a95c32af1ad98ed0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:15,076 INFO [StoreOpener-dad1202300d1ddd3a95c32af1ad98ed0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dad1202300d1ddd3a95c32af1ad98ed0 columnFamilyName C 2024-11-23T15:40:15,076 DEBUG [StoreOpener-dad1202300d1ddd3a95c32af1ad98ed0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:15,077 INFO [StoreOpener-dad1202300d1ddd3a95c32af1ad98ed0-1 {}] regionserver.HStore(327): Store=dad1202300d1ddd3a95c32af1ad98ed0/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:40:15,077 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:15,078 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:15,079 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:15,081 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:15,081 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:15,082 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T15:40:15,084 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:15,088 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:40:15,089 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened dad1202300d1ddd3a95c32af1ad98ed0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60597299, jitterRate=-0.09702987968921661}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T15:40:15,089 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:15,090 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for dad1202300d1ddd3a95c32af1ad98ed0: Running coprocessor pre-open hook at 1732376415063Writing region info on filesystem at 1732376415063Initializing all the Stores at 1732376415065 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376415065Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376415065Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376415065Cleaning up temporary data from old regions at 1732376415081 (+16 ms)Running coprocessor post-open hooks at 1732376415089 (+8 ms)Region opened successfully at 1732376415090 (+1 ms) 2024-11-23T15:40:15,092 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., pid=6, masterSystemTime=1732376415048 2024-11-23T15:40:15,096 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:15,096 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:15,097 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dad1202300d1ddd3a95c32af1ad98ed0, regionState=OPEN, openSeqNum=2, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:15,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 because future has completed 2024-11-23T15:40:15,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T15:40:15,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 in 211 msec 2024-11-23T15:40:15,113 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T15:40:15,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dad1202300d1ddd3a95c32af1ad98ed0, ASSIGN in 377 msec 2024-11-23T15:40:15,116 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T15:40:15,116 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376415116"}]},"ts":"1732376415116"} 2024-11-23T15:40:15,119 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T15:40:15,121 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T15:40:15,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2630 sec 2024-11-23T15:40:16,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T15:40:16,027 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:40:16,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,034 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:40:16,035 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:40:16,037 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:40:16,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:40:16,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:40:16,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d463185, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:40:16,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:40:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:16,042 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:40:16,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14f0fa42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,044 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:40:16,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:40:16,046 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:40:16,048 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:40:16,050 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T15:40:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is b712f9af2c12,42351,1732376408377 2024-11-23T15:40:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@64ba8396 2024-11-23T15:40:16,054 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T15:40:16,056 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T15:40:16,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,061 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:40:16,061 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:40:16,063 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:40:16,063 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:40:16,064 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:40:16,064 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26a9376a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,064 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:40:16,064 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:40:16,065 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:16,066 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:40:16,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a4c56da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,069 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,069 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:40:16,069 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:40:16,071 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:40:16,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:40:16,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:40:16,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6b0c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:40:16,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:40:16,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:16,075 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:40:16,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cdb8e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,079 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,079 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:40:16,079 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:40:16,090 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:40:16,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:40:16,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:40:16,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5676fb0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:40:16,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:40:16,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:16,094 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:40:16,096 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:40:16,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:40:16,099 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:40:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:40:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:40:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:40:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:40:16,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:16,103 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:40:16,105 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253fe80a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,106 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,106 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:40:16,106 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:40:16,107 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:40:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:40:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:40:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3abd488a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:40:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:40:16,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:16,110 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:40:16,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@103656b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,115 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:40:16,115 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:40:16,117 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:40:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:40:16,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:40:16,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bbf9ca7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,118 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:40:16,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:40:16,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:16,120 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:40:16,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19664801, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,122 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,122 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:40:16,123 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:40:16,124 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:40:16,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:40:16,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:40:16,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6396285d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:40:16,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:40:16,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:16,127 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:40:16,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@439f6406, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,131 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,131 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:40:16,131 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:40:16,132 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:40:16,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:40:16,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:40:16,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@92f3d3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:40:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:40:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:16,135 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:40:16,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36bbea39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,138 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,138 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:40:16,138 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:40:16,140 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:40:16,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:40:16,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:40:16,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@463070d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:40:16,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:40:16,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:16,143 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:40:16,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f5c6f22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:16,148 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:40:16,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:40:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:40:16,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:40:16,152 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:40:16,153 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:40:16,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:40:16,155 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:40:16,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:40:16,156 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:40:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:40:16,157 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:40:16,159 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55404, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:40:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:40:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:40:16,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:40:16,162 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:40:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:40:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:40:16,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:40:16,167 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:40:16,167 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:40:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:40:16,169 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:40:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:40:16,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:40:16,171 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:40:16,172 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55428, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:40:16,173 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:40:16,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:40:16,173 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:40:16,173 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:40:16,175 DEBUG [Thread-151 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:40:16,175 DEBUG [Thread-151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:40:16,175 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:40:16,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:40:16,177 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:40:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:40:16,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:40:16,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:40:16,183 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:40:16,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:40:16,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:40:16,188 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:40:16,190 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:40:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:40:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-23T15:40:16,200 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:40:16,202 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:40:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T15:40:16,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:40:16,252 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:40:16,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:16,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T15:40:16,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T15:40:16,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/73a6a0e96b414aaabef28f7c85c56f7c is 50, key is test_row_0/A:col10/1732376416255/Put/seqid=0 2024-11-23T15:40:16,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T15:40:16,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:16,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:16,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:16,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741837_1013 (size=14341) 2024-11-23T15:40:16,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:16,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:16,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/73a6a0e96b414aaabef28f7c85c56f7c 2024-11-23T15:40:16,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:16,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:16,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:16,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376476436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:16,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376476441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:16,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376476454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:16,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376476465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:16,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:16,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376476465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:16,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:16,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T15:40:16,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/1b81c6191be54549b95dd10555d3ff90 is 50, key is test_row_0/B:col10/1732376416255/Put/seqid=0 2024-11-23T15:40:16,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741838_1014 (size=12001) 2024-11-23T15:40:16,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/1b81c6191be54549b95dd10555d3ff90 2024-11-23T15:40:16,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T15:40:16,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376476616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:16,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:16,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:16,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376476618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:16,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:16,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376476617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:16,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:16,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:16,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376476619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:16,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:16,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:16,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:16,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:16,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:16,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376476620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:16,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/de97edf6edbe4e23ac8d427cebdfd0a7 is 50, key is test_row_0/C:col10/1732376416255/Put/seqid=0 2024-11-23T15:40:16,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741839_1015 (size=12001) 2024-11-23T15:40:16,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/de97edf6edbe4e23ac8d427cebdfd0a7 2024-11-23T15:40:16,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/73a6a0e96b414aaabef28f7c85c56f7c as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/73a6a0e96b414aaabef28f7c85c56f7c 2024-11-23T15:40:16,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/73a6a0e96b414aaabef28f7c85c56f7c, entries=200, sequenceid=13, filesize=14.0 K 2024-11-23T15:40:16,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/1b81c6191be54549b95dd10555d3ff90 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1b81c6191be54549b95dd10555d3ff90 2024-11-23T15:40:16,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1b81c6191be54549b95dd10555d3ff90, entries=150, sequenceid=13, filesize=11.7 K 2024-11-23T15:40:16,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/de97edf6edbe4e23ac8d427cebdfd0a7 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/de97edf6edbe4e23ac8d427cebdfd0a7 2024-11-23T15:40:16,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/de97edf6edbe4e23ac8d427cebdfd0a7, entries=150, sequenceid=13, filesize=11.7 K 2024-11-23T15:40:16,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T15:40:16,786 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for dad1202300d1ddd3a95c32af1ad98ed0 in 526ms, sequenceid=13, compaction requested=false 2024-11-23T15:40:16,788 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-23T15:40:16,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:16,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:16,798 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T15:40:16,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/63e61ab664754f788794927cd8ba9664 is 50, key is test_row_0/A:col10/1732376416452/Put/seqid=0 2024-11-23T15:40:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T15:40:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:16,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:16,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741840_1016 (size=12001) 2024-11-23T15:40:16,853 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/63e61ab664754f788794927cd8ba9664 2024-11-23T15:40:16,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:16,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376476860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:16,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:16,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376476856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:16,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:16,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:16,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376476860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:16,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:16,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:16,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/b50660d2fb76453bbed56342e730f28d is 50, key is test_row_0/B:col10/1732376416452/Put/seqid=0 2024-11-23T15:40:16,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376476879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:16,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:16,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376476881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:16,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:16,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:16,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741841_1017 (size=12001) 2024-11-23T15:40:16,897 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/b50660d2fb76453bbed56342e730f28d 2024-11-23T15:40:16,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/4cea9178b818483ea36ba92eddfeb936 is 50, key is test_row_0/C:col10/1732376416452/Put/seqid=0 2024-11-23T15:40:16,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741842_1018 (size=12001) 2024-11-23T15:40:16,949 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/4cea9178b818483ea36ba92eddfeb936 2024-11-23T15:40:16,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/63e61ab664754f788794927cd8ba9664 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/63e61ab664754f788794927cd8ba9664 2024-11-23T15:40:16,985 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/63e61ab664754f788794927cd8ba9664, entries=150, sequenceid=37, filesize=11.7 K 2024-11-23T15:40:16,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/b50660d2fb76453bbed56342e730f28d as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/b50660d2fb76453bbed56342e730f28d 2024-11-23T15:40:16,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:16,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376476985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:16,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:16,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376476986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:16,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:16,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:16,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376476986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:16,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:16,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:17,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376476995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:17,007 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/b50660d2fb76453bbed56342e730f28d, entries=150, sequenceid=37, filesize=11.7 K 2024-11-23T15:40:17,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376476996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:17,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:17,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:17,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/4cea9178b818483ea36ba92eddfeb936 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/4cea9178b818483ea36ba92eddfeb936 2024-11-23T15:40:17,024 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/4cea9178b818483ea36ba92eddfeb936, entries=150, sequenceid=37, filesize=11.7 K 2024-11-23T15:40:17,026 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for dad1202300d1ddd3a95c32af1ad98ed0 in 228ms, sequenceid=37, compaction requested=false 2024-11-23T15:40:17,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:17,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:17,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-23T15:40:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-23T15:40:17,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-23T15:40:17,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 825 msec 2024-11-23T15:40:17,043 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 845 msec 2024-11-23T15:40:17,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:17,238 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:40:17,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T15:40:17,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/5ce8afcb0fcd4c1b963bc638aa8ccb15 is 50, key is test_row_0/A:col10/1732376416855/Put/seqid=0 2024-11-23T15:40:17,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741843_1019 (size=12001) 2024-11-23T15:40:17,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/5ce8afcb0fcd4c1b963bc638aa8ccb15 2024-11-23T15:40:17,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/3c7c59471d014e0b84cb00e8f6e0cf89 is 50, key is test_row_0/B:col10/1732376416855/Put/seqid=0 2024-11-23T15:40:17,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:17,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376477316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:17,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:17,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741844_1020 (size=12001) 2024-11-23T15:40:17,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:17,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376477318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:17,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/3c7c59471d014e0b84cb00e8f6e0cf89 2024-11-23T15:40:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376477322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:17,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:17,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376477323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:17,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T15:40:17,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:17,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376477326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:17,345 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:40:17,346 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:40:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:17,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-23T15:40:17,351 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:40:17,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-23T15:40:17,355 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:40:17,356 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:40:17,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/c64b38f8ebe647aeb3d2659b1f1bf10e is 50, key is test_row_0/C:col10/1732376416855/Put/seqid=0 2024-11-23T15:40:17,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741845_1021 (size=12001) 2024-11-23T15:40:17,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/c64b38f8ebe647aeb3d2659b1f1bf10e 2024-11-23T15:40:17,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/5ce8afcb0fcd4c1b963bc638aa8ccb15 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/5ce8afcb0fcd4c1b963bc638aa8ccb15 2024-11-23T15:40:17,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/5ce8afcb0fcd4c1b963bc638aa8ccb15, entries=150, sequenceid=51, filesize=11.7 K 2024-11-23T15:40:17,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/3c7c59471d014e0b84cb00e8f6e0cf89 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3c7c59471d014e0b84cb00e8f6e0cf89 2024-11-23T15:40:17,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:17,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376477435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:17,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,439 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:17,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:17,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3c7c59471d014e0b84cb00e8f6e0cf89, entries=150, sequenceid=51, filesize=11.7 K 2024-11-23T15:40:17,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376477445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:17,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:17,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:17,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376477446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:17,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/c64b38f8ebe647aeb3d2659b1f1bf10e as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/c64b38f8ebe647aeb3d2659b1f1bf10e 2024-11-23T15:40:17,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376477446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:17,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:17,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:17,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:17,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376477455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:17,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:17,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-23T15:40:17,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/c64b38f8ebe647aeb3d2659b1f1bf10e, entries=150, sequenceid=51, filesize=11.7 K 2024-11-23T15:40:17,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for dad1202300d1ddd3a95c32af1ad98ed0 in 242ms, sequenceid=51, compaction requested=true 2024-11-23T15:40:17,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:17,490 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:40:17,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:17,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:40:17,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:17,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:40:17,494 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:17,496 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:17,500 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:17,501 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:17,503 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/B is initiating minor compaction (all files) 2024-11-23T15:40:17,504 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/B in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:17,504 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1b81c6191be54549b95dd10555d3ff90, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/b50660d2fb76453bbed56342e730f28d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3c7c59471d014e0b84cb00e8f6e0cf89] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=35.2 K 2024-11-23T15:40:17,507 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b81c6191be54549b95dd10555d3ff90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732376416205 2024-11-23T15:40:17,508 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting b50660d2fb76453bbed56342e730f28d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732376416395 2024-11-23T15:40:17,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-23T15:40:17,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:17,512 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-23T15:40:17,513 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c7c59471d014e0b84cb00e8f6e0cf89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732376416853 2024-11-23T15:40:17,521 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:17,521 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/A is initiating minor compaction (all files) 2024-11-23T15:40:17,521 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/A in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:17,522 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/73a6a0e96b414aaabef28f7c85c56f7c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/63e61ab664754f788794927cd8ba9664, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/5ce8afcb0fcd4c1b963bc638aa8ccb15] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=37.4 K 2024-11-23T15:40:17,531 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73a6a0e96b414aaabef28f7c85c56f7c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732376416205 2024-11-23T15:40:17,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/c2ac6e16d6fe4cfc90a4660b6998f9a5 is 50, key is test_row_0/A:col10/1732376417316/Put/seqid=0 2024-11-23T15:40:17,533 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63e61ab664754f788794927cd8ba9664, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732376416395 2024-11-23T15:40:17,535 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ce8afcb0fcd4c1b963bc638aa8ccb15, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732376416853 2024-11-23T15:40:17,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741846_1022 (size=12001) 2024-11-23T15:40:17,587 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/c2ac6e16d6fe4cfc90a4660b6998f9a5 2024-11-23T15:40:17,595 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#A#compaction#10 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:17,596 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/efc6113e26a64e20b9fd047d06966d60 is 50, key is test_row_0/A:col10/1732376416855/Put/seqid=0 2024-11-23T15:40:17,628 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#B#compaction#11 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:17,629 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/9544fc4582274bb6aecf607718f54c1b is 50, key is test_row_0/B:col10/1732376416855/Put/seqid=0 2024-11-23T15:40:17,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/8df211a484e04a318fbf5ceff704e9db is 50, key is test_row_0/B:col10/1732376417316/Put/seqid=0 2024-11-23T15:40:17,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-23T15:40:17,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741849_1025 (size=12001) 2024-11-23T15:40:17,686 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/8df211a484e04a318fbf5ceff704e9db 2024-11-23T15:40:17,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741847_1023 (size=12104) 2024-11-23T15:40:17,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741848_1024 (size=12104) 2024-11-23T15:40:17,704 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/efc6113e26a64e20b9fd047d06966d60 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/efc6113e26a64e20b9fd047d06966d60 2024-11-23T15:40:17,714 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/9544fc4582274bb6aecf607718f54c1b as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/9544fc4582274bb6aecf607718f54c1b 2024-11-23T15:40:17,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/48dfa0c4c0a847428caa7d3464455598 is 50, key is test_row_0/C:col10/1732376417316/Put/seqid=0 2024-11-23T15:40:17,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741850_1026 (size=12001) 2024-11-23T15:40:17,734 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/48dfa0c4c0a847428caa7d3464455598 2024-11-23T15:40:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:17,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:17,740 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/A of dad1202300d1ddd3a95c32af1ad98ed0 into efc6113e26a64e20b9fd047d06966d60(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:17,741 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/B of dad1202300d1ddd3a95c32af1ad98ed0 into 9544fc4582274bb6aecf607718f54c1b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:17,741 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:17,740 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:17,745 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/A, priority=13, startTime=1732376417482; duration=0sec 2024-11-23T15:40:17,745 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:17,746 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:A 2024-11-23T15:40:17,746 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:17,747 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/B, priority=13, startTime=1732376417493; duration=0sec 2024-11-23T15:40:17,747 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:17,747 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:B 2024-11-23T15:40:17,749 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:17,749 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/C is initiating minor compaction (all files) 2024-11-23T15:40:17,750 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/C in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:17,750 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/de97edf6edbe4e23ac8d427cebdfd0a7, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/4cea9178b818483ea36ba92eddfeb936, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/c64b38f8ebe647aeb3d2659b1f1bf10e] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=35.2 K 2024-11-23T15:40:17,752 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting de97edf6edbe4e23ac8d427cebdfd0a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732376416205 2024-11-23T15:40:17,758 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cea9178b818483ea36ba92eddfeb936, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732376416395 2024-11-23T15:40:17,760 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting c64b38f8ebe647aeb3d2659b1f1bf10e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732376416853 2024-11-23T15:40:17,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/c2ac6e16d6fe4cfc90a4660b6998f9a5 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/c2ac6e16d6fe4cfc90a4660b6998f9a5 2024-11-23T15:40:17,786 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/c2ac6e16d6fe4cfc90a4660b6998f9a5, entries=150, sequenceid=73, filesize=11.7 K 2024-11-23T15:40:17,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/8df211a484e04a318fbf5ceff704e9db as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/8df211a484e04a318fbf5ceff704e9db 2024-11-23T15:40:17,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:17,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376477778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:17,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,793 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:17,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376477784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:17,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:17,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376477788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:17,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:17,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376477788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:17,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:17,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:17,812 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#C#compaction#14 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:17,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376477793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:17,813 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/af0203bf575b455fae81cc7024f60e16 is 50, key is test_row_0/C:col10/1732376416855/Put/seqid=0 2024-11-23T15:40:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:17,817 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/8df211a484e04a318fbf5ceff704e9db, entries=150, sequenceid=73, filesize=11.7 K 2024-11-23T15:40:17,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/48dfa0c4c0a847428caa7d3464455598 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/48dfa0c4c0a847428caa7d3464455598 2024-11-23T15:40:17,833 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/48dfa0c4c0a847428caa7d3464455598, entries=150, sequenceid=73, filesize=11.7 K 2024-11-23T15:40:17,837 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for dad1202300d1ddd3a95c32af1ad98ed0 in 324ms, sequenceid=73, compaction requested=false 2024-11-23T15:40:17,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:17,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:17,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-23T15:40:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-23T15:40:17,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-23T15:40:17,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 484 msec 2024-11-23T15:40:17,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 504 msec 2024-11-23T15:40:17,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741851_1027 (size=12104) 2024-11-23T15:40:17,878 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/af0203bf575b455fae81cc7024f60e16 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/af0203bf575b455fae81cc7024f60e16 2024-11-23T15:40:17,902 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/C of dad1202300d1ddd3a95c32af1ad98ed0 into af0203bf575b455fae81cc7024f60e16(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:17,902 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:17,902 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/C, priority=13, startTime=1732376417494; duration=0sec 2024-11-23T15:40:17,903 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:17,903 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:C 2024-11-23T15:40:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:17,919 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:40:17,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-23T15:40:17,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/a8818f386cfd40cc8392621c57571876 is 50, key is test_row_0/A:col10/1732376417738/Put/seqid=0 2024-11-23T15:40:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-23T15:40:17,986 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:40:17,986 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:40:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-23T15:40:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T15:40:17,990 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:40:17,992 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:40:17,992 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:40:18,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741852_1028 (size=14341) 2024-11-23T15:40:18,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/a8818f386cfd40cc8392621c57571876 2024-11-23T15:40:18,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/447159a9bd3346ed861775dd6856c8fb is 50, key is test_row_0/B:col10/1732376417738/Put/seqid=0 2024-11-23T15:40:18,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376478022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376478030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376478036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376478033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376478036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741853_1029 (size=12001) 2024-11-23T15:40:18,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/447159a9bd3346ed861775dd6856c8fb 2024-11-23T15:40:18,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T15:40:18,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/95b835c6375a4e77988cceb7db2edfef is 50, key is test_row_0/C:col10/1732376417738/Put/seqid=0 2024-11-23T15:40:18,118 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-23T15:40:18,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741854_1030 (size=12001) 2024-11-23T15:40:18,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/95b835c6375a4e77988cceb7db2edfef 2024-11-23T15:40:18,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376478145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-23T15:40:18,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:18,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:18,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:18,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:18,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:18,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/a8818f386cfd40cc8392621c57571876 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/a8818f386cfd40cc8392621c57571876 2024-11-23T15:40:18,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376478166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376478166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376478166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376478166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/a8818f386cfd40cc8392621c57571876, entries=200, sequenceid=91, filesize=14.0 K 2024-11-23T15:40:18,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/447159a9bd3346ed861775dd6856c8fb as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/447159a9bd3346ed861775dd6856c8fb 2024-11-23T15:40:18,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/447159a9bd3346ed861775dd6856c8fb, entries=150, sequenceid=91, filesize=11.7 K 2024-11-23T15:40:18,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/95b835c6375a4e77988cceb7db2edfef as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/95b835c6375a4e77988cceb7db2edfef 2024-11-23T15:40:18,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/95b835c6375a4e77988cceb7db2edfef, entries=150, sequenceid=91, filesize=11.7 K 2024-11-23T15:40:18,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for dad1202300d1ddd3a95c32af1ad98ed0 in 292ms, sequenceid=91, compaction requested=true 2024-11-23T15:40:18,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:18,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:40:18,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:18,213 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:18,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:40:18,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:18,213 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:18,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:40:18,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:18,216 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:18,216 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/B is initiating minor compaction (all files) 2024-11-23T15:40:18,216 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/B in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:18,217 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/9544fc4582274bb6aecf607718f54c1b, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/8df211a484e04a318fbf5ceff704e9db, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/447159a9bd3346ed861775dd6856c8fb] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=35.3 K 2024-11-23T15:40:18,217 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:18,217 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/A is initiating minor compaction (all files) 2024-11-23T15:40:18,217 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/A in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:18,217 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/efc6113e26a64e20b9fd047d06966d60, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/c2ac6e16d6fe4cfc90a4660b6998f9a5, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/a8818f386cfd40cc8392621c57571876] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=37.5 K 2024-11-23T15:40:18,218 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 9544fc4582274bb6aecf607718f54c1b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732376416853 2024-11-23T15:40:18,219 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting efc6113e26a64e20b9fd047d06966d60, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732376416853 2024-11-23T15:40:18,220 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8df211a484e04a318fbf5ceff704e9db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732376417310 2024-11-23T15:40:18,221 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 447159a9bd3346ed861775dd6856c8fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732376417738 2024-11-23T15:40:18,222 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2ac6e16d6fe4cfc90a4660b6998f9a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732376417310 2024-11-23T15:40:18,223 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8818f386cfd40cc8392621c57571876, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732376417738 2024-11-23T15:40:18,251 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#B#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:18,252 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/eacec568583e4c3582a5598b21a7623f is 50, key is test_row_0/B:col10/1732376417738/Put/seqid=0 2024-11-23T15:40:18,266 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:18,267 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/6069b726b76946d89881845a19a77f71 is 50, key is test_row_0/A:col10/1732376417738/Put/seqid=0 2024-11-23T15:40:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T15:40:18,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-23T15:40:18,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:18,310 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-23T15:40:18,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741855_1031 (size=12207) 2024-11-23T15:40:18,330 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/eacec568583e4c3582a5598b21a7623f as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/eacec568583e4c3582a5598b21a7623f 2024-11-23T15:40:18,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741856_1032 (size=12207) 2024-11-23T15:40:18,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/0b3291f0ab5045a3b3ed725fcd5a92f2 is 50, key is test_row_0/A:col10/1732376418029/Put/seqid=0 2024-11-23T15:40:18,346 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/6069b726b76946d89881845a19a77f71 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6069b726b76946d89881845a19a77f71 2024-11-23T15:40:18,351 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/B of dad1202300d1ddd3a95c32af1ad98ed0 into eacec568583e4c3582a5598b21a7623f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:18,351 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:18,351 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/B, priority=13, startTime=1732376418213; duration=0sec 2024-11-23T15:40:18,352 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:18,352 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:B 2024-11-23T15:40:18,352 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:18,358 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:18,358 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/C is initiating minor compaction (all files) 2024-11-23T15:40:18,358 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/C in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:18,359 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/af0203bf575b455fae81cc7024f60e16, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/48dfa0c4c0a847428caa7d3464455598, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/95b835c6375a4e77988cceb7db2edfef] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=35.3 K 2024-11-23T15:40:18,361 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting af0203bf575b455fae81cc7024f60e16, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732376416853 2024-11-23T15:40:18,362 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 48dfa0c4c0a847428caa7d3464455598, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732376417310 2024-11-23T15:40:18,363 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/A of dad1202300d1ddd3a95c32af1ad98ed0 into 6069b726b76946d89881845a19a77f71(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:18,363 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 95b835c6375a4e77988cceb7db2edfef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732376417738 2024-11-23T15:40:18,363 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:18,363 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/A, priority=13, startTime=1732376418213; duration=0sec 2024-11-23T15:40:18,363 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:18,364 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:A 2024-11-23T15:40:18,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741857_1033 (size=12001) 2024-11-23T15:40:18,374 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/0b3291f0ab5045a3b3ed725fcd5a92f2 2024-11-23T15:40:18,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:18,398 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:18,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/fb1b1e13127d4e42b485d7d7ede1ac98 is 50, key is test_row_0/B:col10/1732376418029/Put/seqid=0 2024-11-23T15:40:18,399 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/afe37d6bae39494dad7b86cb834b5bee is 50, key is test_row_0/C:col10/1732376417738/Put/seqid=0 2024-11-23T15:40:18,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376478419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376478422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376478425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376478431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376478439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741858_1034 (size=12207) 2024-11-23T15:40:18,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741859_1035 (size=12001) 2024-11-23T15:40:18,479 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/fb1b1e13127d4e42b485d7d7ede1ac98 2024-11-23T15:40:18,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/b1ee220b8bdd4c2bab2145ca65c425f1 is 50, key is test_row_0/C:col10/1732376418029/Put/seqid=0 2024-11-23T15:40:18,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376478534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741860_1036 (size=12001) 2024-11-23T15:40:18,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,552 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376478545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376478545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,555 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,561 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/b1ee220b8bdd4c2bab2145ca65c425f1 2024-11-23T15:40:18,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376478554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376478558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/0b3291f0ab5045a3b3ed725fcd5a92f2 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/0b3291f0ab5045a3b3ed725fcd5a92f2 2024-11-23T15:40:18,596 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/0b3291f0ab5045a3b3ed725fcd5a92f2, entries=150, sequenceid=113, filesize=11.7 K 2024-11-23T15:40:18,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/fb1b1e13127d4e42b485d7d7ede1ac98 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fb1b1e13127d4e42b485d7d7ede1ac98 2024-11-23T15:40:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T15:40:18,621 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fb1b1e13127d4e42b485d7d7ede1ac98, entries=150, sequenceid=113, filesize=11.7 K 2024-11-23T15:40:18,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/b1ee220b8bdd4c2bab2145ca65c425f1 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/b1ee220b8bdd4c2bab2145ca65c425f1 2024-11-23T15:40:18,639 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/b1ee220b8bdd4c2bab2145ca65c425f1, entries=150, sequenceid=113, filesize=11.7 K 2024-11-23T15:40:18,643 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for dad1202300d1ddd3a95c32af1ad98ed0 in 331ms, sequenceid=113, compaction requested=false 2024-11-23T15:40:18,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:18,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:18,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-23T15:40:18,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-23T15:40:18,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-23T15:40:18,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 654 msec 2024-11-23T15:40:18,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 671 msec 2024-11-23T15:40:18,759 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:40:18,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-23T15:40:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:18,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/583060b9da95457a8f75305020fc3691 is 50, key is test_row_0/A:col10/1732376418425/Put/seqid=0 2024-11-23T15:40:18,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741861_1037 (size=14441) 2024-11-23T15:40:18,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376478823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376478828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376478832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376478835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376478827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,876 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/afe37d6bae39494dad7b86cb834b5bee as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/afe37d6bae39494dad7b86cb834b5bee 2024-11-23T15:40:18,887 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/C of dad1202300d1ddd3a95c32af1ad98ed0 into afe37d6bae39494dad7b86cb834b5bee(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:18,887 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:18,887 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/C, priority=13, startTime=1732376418213; duration=0sec 2024-11-23T15:40:18,888 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:18,888 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:C 2024-11-23T15:40:18,913 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T15:40:18,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376478944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376478944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376478955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376478955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,960 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376478955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:18,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:18,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:18,990 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-23T15:40:18,991 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-23T15:40:18,993 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T15:40:18,993 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-23T15:40:18,994 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-23T15:40:18,994 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-23T15:40:18,995 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-23T15:40:18,995 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-23T15:40:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T15:40:19,125 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:40:19,127 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:40:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-23T15:40:19,132 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:40:19,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T15:40:19,134 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:40:19,134 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:40:19,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376479155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:19,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:19,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:19,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:19,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:19,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376479155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:19,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:19,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:19,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:19,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:19,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:19,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376479166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:19,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376479166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:19,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:19,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376479166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:19,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:19,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:19,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:19,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/583060b9da95457a8f75305020fc3691 2024-11-23T15:40:19,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/7501669030644e41808ba47ca82680a0 is 50, key is test_row_0/B:col10/1732376418425/Put/seqid=0 2024-11-23T15:40:19,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T15:40:19,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741862_1038 (size=12101) 2024-11-23T15:40:19,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/7501669030644e41808ba47ca82680a0 2024-11-23T15:40:19,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T15:40:19,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:19,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:19,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:19,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:19,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:19,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:19,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/fef07cada93743258969cad3113f0f70 is 50, key is test_row_0/C:col10/1732376418425/Put/seqid=0 2024-11-23T15:40:19,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741863_1039 (size=12101) 2024-11-23T15:40:19,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T15:40:19,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:19,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:19,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:19,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:19,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T15:40:19,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:19,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376479475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:19,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:19,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:19,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:19,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376479475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:19,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:19,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:19,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:19,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:19,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376479475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:19,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376479476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:19,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376479478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:19,480 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:19,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T15:40:19,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:19,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:19,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:19,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:19,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:19,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:19,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/fef07cada93743258969cad3113f0f70 2024-11-23T15:40:19,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/583060b9da95457a8f75305020fc3691 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/583060b9da95457a8f75305020fc3691 2024-11-23T15:40:19,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/583060b9da95457a8f75305020fc3691, entries=200, sequenceid=131, filesize=14.1 K 2024-11-23T15:40:19,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/7501669030644e41808ba47ca82680a0 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/7501669030644e41808ba47ca82680a0 2024-11-23T15:40:19,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T15:40:19,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:19,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:19,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:19,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:19,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:19,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:19,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T15:40:19,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/7501669030644e41808ba47ca82680a0, entries=150, sequenceid=131, filesize=11.8 K 2024-11-23T15:40:19,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/fef07cada93743258969cad3113f0f70 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/fef07cada93743258969cad3113f0f70 2024-11-23T15:40:19,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/fef07cada93743258969cad3113f0f70, entries=150, sequenceid=131, filesize=11.8 K 2024-11-23T15:40:19,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for dad1202300d1ddd3a95c32af1ad98ed0 in 1031ms, sequenceid=131, compaction requested=true 2024-11-23T15:40:19,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:19,791 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:19,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:40:19,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:19,792 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:19,793 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/A is initiating minor compaction (all files) 2024-11-23T15:40:19,793 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/A in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:19,793 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6069b726b76946d89881845a19a77f71, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/0b3291f0ab5045a3b3ed725fcd5a92f2, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/583060b9da95457a8f75305020fc3691] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=37.7 K 2024-11-23T15:40:19,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:40:19,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:19,793 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:19,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:40:19,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:19,794 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6069b726b76946d89881845a19a77f71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732376417738 2024-11-23T15:40:19,795 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b3291f0ab5045a3b3ed725fcd5a92f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732376418025 2024-11-23T15:40:19,797 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:19,798 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/B is initiating minor compaction (all files) 2024-11-23T15:40:19,798 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 583060b9da95457a8f75305020fc3691, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732376418407 2024-11-23T15:40:19,798 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/B in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:19,798 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/eacec568583e4c3582a5598b21a7623f, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fb1b1e13127d4e42b485d7d7ede1ac98, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/7501669030644e41808ba47ca82680a0] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=35.5 K 2024-11-23T15:40:19,800 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting eacec568583e4c3582a5598b21a7623f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732376417738 2024-11-23T15:40:19,801 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting fb1b1e13127d4e42b485d7d7ede1ac98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732376418025 2024-11-23T15:40:19,802 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 7501669030644e41808ba47ca82680a0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732376418425 2024-11-23T15:40:19,829 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#A#compaction#27 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:19,830 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/7b430c6731414f9fae66bc4cd9949432 is 50, key is test_row_0/A:col10/1732376418425/Put/seqid=0 2024-11-23T15:40:19,843 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:19,844 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/0b0f88c84c1c43c2a838280475e7a174 is 50, key is test_row_0/B:col10/1732376418425/Put/seqid=0 2024-11-23T15:40:19,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741864_1040 (size=12409) 2024-11-23T15:40:19,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741865_1041 (size=12409) 2024-11-23T15:40:19,900 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/7b430c6731414f9fae66bc4cd9949432 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/7b430c6731414f9fae66bc4cd9949432 2024-11-23T15:40:19,923 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/0b0f88c84c1c43c2a838280475e7a174 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/0b0f88c84c1c43c2a838280475e7a174 2024-11-23T15:40:19,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T15:40:19,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:19,925 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-23T15:40:19,934 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/A of dad1202300d1ddd3a95c32af1ad98ed0 into 7b430c6731414f9fae66bc4cd9949432(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:19,934 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:19,934 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/A, priority=13, startTime=1732376419790; duration=0sec 2024-11-23T15:40:19,935 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:19,935 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:A 2024-11-23T15:40:19,935 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:19,939 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:19,939 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/C is initiating minor compaction (all files) 2024-11-23T15:40:19,939 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/C in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:19,939 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/afe37d6bae39494dad7b86cb834b5bee, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/b1ee220b8bdd4c2bab2145ca65c425f1, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/fef07cada93743258969cad3113f0f70] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=35.5 K 2024-11-23T15:40:19,940 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting afe37d6bae39494dad7b86cb834b5bee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732376417738 2024-11-23T15:40:19,941 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1ee220b8bdd4c2bab2145ca65c425f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732376418025 2024-11-23T15:40:19,941 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/B of dad1202300d1ddd3a95c32af1ad98ed0 into 0b0f88c84c1c43c2a838280475e7a174(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:19,942 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:19,942 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/B, priority=13, startTime=1732376419793; duration=0sec 2024-11-23T15:40:19,942 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting fef07cada93743258969cad3113f0f70, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732376418425 2024-11-23T15:40:19,942 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:19,943 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:B 2024-11-23T15:40:19,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/6bf7154f813347ee9c5428783b1fc441 is 50, key is test_row_0/A:col10/1732376418831/Put/seqid=0 2024-11-23T15:40:19,989 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#C#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:19,990 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/8e3ab70751d542a5be8db0e022cb3e88 is 50, key is test_row_0/C:col10/1732376418425/Put/seqid=0 2024-11-23T15:40:19,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:19,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:20,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741866_1042 (size=12151) 2024-11-23T15:40:20,036 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/6bf7154f813347ee9c5428783b1fc441 2024-11-23T15:40:20,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:20,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376480021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:20,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:20,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376480031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:20,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:20,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:20,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376480035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:20,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:20,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376480037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:20,050 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:20,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:20,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:20,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376480046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:20,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:20,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/3580e0c343094aff9ef27438660b356c is 50, key is test_row_0/B:col10/1732376418831/Put/seqid=0 2024-11-23T15:40:20,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741867_1043 (size=12409) 2024-11-23T15:40:20,074 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/8e3ab70751d542a5be8db0e022cb3e88 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/8e3ab70751d542a5be8db0e022cb3e88 2024-11-23T15:40:20,087 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/C of dad1202300d1ddd3a95c32af1ad98ed0 into 8e3ab70751d542a5be8db0e022cb3e88(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:20,087 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:20,087 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/C, priority=13, startTime=1732376419793; duration=0sec 2024-11-23T15:40:20,087 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:20,088 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:C 2024-11-23T15:40:20,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741868_1044 (size=12151) 2024-11-23T15:40:20,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376480147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:20,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:20,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376480147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:20,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:20,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:20,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376480155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:20,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:20,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:20,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376480155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:20,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:20,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376480164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:20,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T15:40:20,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376480365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:20,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376480365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:20,373 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:20,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376480375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:20,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376480377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:20,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376480394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:20,522 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/3580e0c343094aff9ef27438660b356c 2024-11-23T15:40:20,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/3a86b3fb212741bd8e6f7a3f621e1f86 is 50, key is test_row_0/C:col10/1732376418831/Put/seqid=0 2024-11-23T15:40:20,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741869_1045 (size=12151) 2024-11-23T15:40:20,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:20,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376480685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:20,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376480685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:20,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376480685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:20,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:20,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:20,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:20,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376480695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:20,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:20,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376480714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:20,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:20,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:20,998 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/3a86b3fb212741bd8e6f7a3f621e1f86 2024-11-23T15:40:21,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/6bf7154f813347ee9c5428783b1fc441 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6bf7154f813347ee9c5428783b1fc441 2024-11-23T15:40:21,037 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6bf7154f813347ee9c5428783b1fc441, entries=150, sequenceid=152, filesize=11.9 K 2024-11-23T15:40:21,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/3580e0c343094aff9ef27438660b356c as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3580e0c343094aff9ef27438660b356c 2024-11-23T15:40:21,063 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3580e0c343094aff9ef27438660b356c, entries=150, sequenceid=152, filesize=11.9 K 2024-11-23T15:40:21,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/3a86b3fb212741bd8e6f7a3f621e1f86 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/3a86b3fb212741bd8e6f7a3f621e1f86 2024-11-23T15:40:21,084 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/3a86b3fb212741bd8e6f7a3f621e1f86, entries=150, sequenceid=152, filesize=11.9 K 2024-11-23T15:40:21,099 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for dad1202300d1ddd3a95c32af1ad98ed0 in 1174ms, sequenceid=152, compaction requested=false 2024-11-23T15:40:21,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:21,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:21,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-23T15:40:21,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-23T15:40:21,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-23T15:40:21,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9690 sec 2024-11-23T15:40:21,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.9820 sec 2024-11-23T15:40:21,202 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:40:21,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-23T15:40:21,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:21,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/bf69bc0559e14ae6b31d974493578ad2 is 50, key is test_row_0/A:col10/1732376419995/Put/seqid=0 2024-11-23T15:40:21,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741870_1046 (size=14541) 2024-11-23T15:40:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T15:40:21,285 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:40:21,286 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:40:21,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-23T15:40:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-23T15:40:21,290 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:40:21,291 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:40:21,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:40:21,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:21,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376481314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:21,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:21,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:21,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376481314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:21,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:21,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:21,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376481319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:21,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376481322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:21,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:21,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376481325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:21,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-23T15:40:21,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376481434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376481434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:21,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376481434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:21,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-23T15:40:21,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:21,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:21,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:21,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:21,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:21,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:21,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376481445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:21,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376481445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:21,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:21,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:21,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-23T15:40:21,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:21,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:21,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:21,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:21,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:21,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-23T15:40:21,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376481645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:21,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376481645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,655 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:21,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376481645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:21,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:21,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376481655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:21,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376481655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:21,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/bf69bc0559e14ae6b31d974493578ad2 2024-11-23T15:40:21,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/3f61fb5c7b3b4f92a29386d8166f1b7f is 50, key is test_row_0/B:col10/1732376419995/Put/seqid=0 2024-11-23T15:40:21,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741871_1047 (size=12151) 2024-11-23T15:40:21,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/3f61fb5c7b3b4f92a29386d8166f1b7f 2024-11-23T15:40:21,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/b2c59c35ff99486b926603779d00b47a is 50, key is test_row_0/C:col10/1732376419995/Put/seqid=0 2024-11-23T15:40:21,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-23T15:40:21,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:21,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:21,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:21,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:21,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:21,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741872_1048 (size=12151) 2024-11-23T15:40:21,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/b2c59c35ff99486b926603779d00b47a 2024-11-23T15:40:21,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/bf69bc0559e14ae6b31d974493578ad2 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/bf69bc0559e14ae6b31d974493578ad2 2024-11-23T15:40:21,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/bf69bc0559e14ae6b31d974493578ad2, entries=200, sequenceid=172, filesize=14.2 K 2024-11-23T15:40:21,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/3f61fb5c7b3b4f92a29386d8166f1b7f as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3f61fb5c7b3b4f92a29386d8166f1b7f 2024-11-23T15:40:21,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3f61fb5c7b3b4f92a29386d8166f1b7f, entries=150, sequenceid=172, filesize=11.9 K 2024-11-23T15:40:21,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/b2c59c35ff99486b926603779d00b47a as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/b2c59c35ff99486b926603779d00b47a 2024-11-23T15:40:21,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/b2c59c35ff99486b926603779d00b47a, entries=150, sequenceid=172, filesize=11.9 K 2024-11-23T15:40:21,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for dad1202300d1ddd3a95c32af1ad98ed0 in 676ms, sequenceid=172, compaction requested=true 2024-11-23T15:40:21,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:21,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:40:21,879 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:21,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:21,879 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:21,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:40:21,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:21,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:40:21,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:21,881 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39101 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:21,881 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:21,881 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/A is initiating minor compaction (all files) 2024-11-23T15:40:21,881 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/B is initiating minor compaction (all files) 2024-11-23T15:40:21,881 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/A in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:21,881 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/B in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:21,882 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/0b0f88c84c1c43c2a838280475e7a174, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3580e0c343094aff9ef27438660b356c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3f61fb5c7b3b4f92a29386d8166f1b7f] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=35.9 K 2024-11-23T15:40:21,882 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/7b430c6731414f9fae66bc4cd9949432, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6bf7154f813347ee9c5428783b1fc441, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/bf69bc0559e14ae6b31d974493578ad2] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=38.2 K 2024-11-23T15:40:21,882 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b430c6731414f9fae66bc4cd9949432, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732376418425 2024-11-23T15:40:21,882 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b0f88c84c1c43c2a838280475e7a174, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732376418425 2024-11-23T15:40:21,883 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bf7154f813347ee9c5428783b1fc441, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732376418824 2024-11-23T15:40:21,883 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 3580e0c343094aff9ef27438660b356c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732376418824 2024-11-23T15:40:21,884 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f61fb5c7b3b4f92a29386d8166f1b7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732376419995 2024-11-23T15:40:21,884 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf69bc0559e14ae6b31d974493578ad2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732376419995 2024-11-23T15:40:21,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-23T15:40:21,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:21,912 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-23T15:40:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-23T15:40:21,915 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#B#compaction#36 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:21,916 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/81e8569b11b34244bc3df0fd238d0c6f is 50, key is test_row_0/B:col10/1732376419995/Put/seqid=0 2024-11-23T15:40:21,916 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#A#compaction#37 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:21,917 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/7324e307a8d74712ad2ac36111309262 is 50, key is test_row_0/A:col10/1732376419995/Put/seqid=0 2024-11-23T15:40:21,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/1e6717fb19044bd0822a44639c0955fe is 50, key is test_row_0/A:col10/1732376421293/Put/seqid=0 2024-11-23T15:40:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:21,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:21,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741873_1049 (size=12561) 2024-11-23T15:40:21,992 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/81e8569b11b34244bc3df0fd238d0c6f as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/81e8569b11b34244bc3df0fd238d0c6f 2024-11-23T15:40:22,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741874_1050 (size=12561) 2024-11-23T15:40:22,020 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/7324e307a8d74712ad2ac36111309262 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/7324e307a8d74712ad2ac36111309262 2024-11-23T15:40:22,022 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/B of dad1202300d1ddd3a95c32af1ad98ed0 into 81e8569b11b34244bc3df0fd238d0c6f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:22,022 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:22,022 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/B, priority=13, startTime=1732376421879; duration=0sec 2024-11-23T15:40:22,022 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:22,022 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:B 2024-11-23T15:40:22,022 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:22,025 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:22,025 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/C is initiating minor compaction (all files) 2024-11-23T15:40:22,026 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/C in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:22,027 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/8e3ab70751d542a5be8db0e022cb3e88, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/3a86b3fb212741bd8e6f7a3f621e1f86, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/b2c59c35ff99486b926603779d00b47a] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=35.9 K 2024-11-23T15:40:22,028 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e3ab70751d542a5be8db0e022cb3e88, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732376418425 2024-11-23T15:40:22,032 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/A of dad1202300d1ddd3a95c32af1ad98ed0 into 7324e307a8d74712ad2ac36111309262(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:22,032 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:22,032 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/A, priority=13, startTime=1732376421879; duration=0sec 2024-11-23T15:40:22,032 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:22,032 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:A 2024-11-23T15:40:22,033 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a86b3fb212741bd8e6f7a3f621e1f86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732376418824 2024-11-23T15:40:22,034 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting b2c59c35ff99486b926603779d00b47a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732376419995 2024-11-23T15:40:22,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741875_1051 (size=12151) 2024-11-23T15:40:22,037 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/1e6717fb19044bd0822a44639c0955fe 2024-11-23T15:40:22,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/cdbf86d562a0472d8362d517e72ee77b is 50, key is test_row_0/B:col10/1732376421293/Put/seqid=0 2024-11-23T15:40:22,076 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#C#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:22,077 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/2bb8ca1a852c48b3ba2bc19f8f299d68 is 50, key is test_row_0/C:col10/1732376419995/Put/seqid=0 2024-11-23T15:40:22,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376482069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376482074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376482085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376482088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376482088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741877_1053 (size=12561) 2024-11-23T15:40:22,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741876_1052 (size=12151) 2024-11-23T15:40:22,128 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/cdbf86d562a0472d8362d517e72ee77b 2024-11-23T15:40:22,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/d432d2037fdb4454acf020761bcd66b6 is 50, key is test_row_0/C:col10/1732376421293/Put/seqid=0 2024-11-23T15:40:22,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741878_1054 (size=12151) 2024-11-23T15:40:22,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376482194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376482195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376482195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376482195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376482204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,208 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/d432d2037fdb4454acf020761bcd66b6 2024-11-23T15:40:22,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/1e6717fb19044bd0822a44639c0955fe as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/1e6717fb19044bd0822a44639c0955fe 2024-11-23T15:40:22,253 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/1e6717fb19044bd0822a44639c0955fe, entries=150, sequenceid=192, filesize=11.9 K 2024-11-23T15:40:22,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/cdbf86d562a0472d8362d517e72ee77b as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/cdbf86d562a0472d8362d517e72ee77b 2024-11-23T15:40:22,274 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/cdbf86d562a0472d8362d517e72ee77b, entries=150, sequenceid=192, filesize=11.9 K 2024-11-23T15:40:22,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/d432d2037fdb4454acf020761bcd66b6 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/d432d2037fdb4454acf020761bcd66b6 2024-11-23T15:40:22,290 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/d432d2037fdb4454acf020761bcd66b6, entries=150, sequenceid=192, filesize=11.9 K 2024-11-23T15:40:22,292 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for dad1202300d1ddd3a95c32af1ad98ed0 in 381ms, sequenceid=192, compaction requested=false 2024-11-23T15:40:22,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:22,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:22,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-23T15:40:22,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-23T15:40:22,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-23T15:40:22,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0080 sec 2024-11-23T15:40:22,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.0290 sec 2024-11-23T15:40:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:22,408 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:40:22,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-23T15:40:22,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/76b098206e8e417fb1b119a832ca53b7 is 50, key is test_row_0/A:col10/1732376422406/Put/seqid=0 2024-11-23T15:40:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-23T15:40:22,426 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:40:22,428 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:40:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-23T15:40:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T15:40:22,432 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:40:22,433 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:40:22,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:40:22,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376482439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376482444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741879_1055 (size=16931) 2024-11-23T15:40:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376482448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376482448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376482448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/76b098206e8e417fb1b119a832ca53b7 2024-11-23T15:40:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/d336e825e46a45b7b134496887cdcfeb is 50, key is test_row_0/B:col10/1732376422406/Put/seqid=0 2024-11-23T15:40:22,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741880_1056 (size=12151) 2024-11-23T15:40:22,519 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/2bb8ca1a852c48b3ba2bc19f8f299d68 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/2bb8ca1a852c48b3ba2bc19f8f299d68 2024-11-23T15:40:22,528 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/C of dad1202300d1ddd3a95c32af1ad98ed0 into 2bb8ca1a852c48b3ba2bc19f8f299d68(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:22,529 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:22,529 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/C, priority=13, startTime=1732376421880; duration=0sec 2024-11-23T15:40:22,529 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:22,529 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:C 2024-11-23T15:40:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T15:40:22,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376482555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,558 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376482565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376482566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376482566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,572 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376482566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-23T15:40:22,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:22,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:22,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:22,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:22,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:22,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T15:40:22,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-23T15:40:22,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:22,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:22,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:22,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:22,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:22,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376482765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376482774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376482785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376482785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376482785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:22,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:22,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/d336e825e46a45b7b134496887cdcfeb 2024-11-23T15:40:22,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-23T15:40:22,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:22,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:22,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:22,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:22,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:22,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/941bb1dd86bb47fbb6a6f5b1764162e5 is 50, key is test_row_0/C:col10/1732376422406/Put/seqid=0 2024-11-23T15:40:22,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741881_1057 (size=12151) 2024-11-23T15:40:22,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/941bb1dd86bb47fbb6a6f5b1764162e5 2024-11-23T15:40:22,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/76b098206e8e417fb1b119a832ca53b7 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/76b098206e8e417fb1b119a832ca53b7 2024-11-23T15:40:22,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/76b098206e8e417fb1b119a832ca53b7, entries=250, sequenceid=213, filesize=16.5 K 2024-11-23T15:40:22,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/d336e825e46a45b7b134496887cdcfeb as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/d336e825e46a45b7b134496887cdcfeb 2024-11-23T15:40:22,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/d336e825e46a45b7b134496887cdcfeb, entries=150, sequenceid=213, filesize=11.9 K 2024-11-23T15:40:22,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/941bb1dd86bb47fbb6a6f5b1764162e5 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/941bb1dd86bb47fbb6a6f5b1764162e5 2024-11-23T15:40:22,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/941bb1dd86bb47fbb6a6f5b1764162e5, entries=150, sequenceid=213, filesize=11.9 K 2024-11-23T15:40:22,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for dad1202300d1ddd3a95c32af1ad98ed0 in 553ms, sequenceid=213, compaction requested=true 2024-11-23T15:40:22,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:22,961 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:22,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:40:22,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:22,962 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:22,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:40:22,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:22,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:40:22,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:22,963 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:22,963 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/A is initiating minor compaction (all files) 2024-11-23T15:40:22,963 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:22,963 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/B is initiating minor compaction (all files) 2024-11-23T15:40:22,963 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/A in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:22,963 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/B in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:22,963 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/7324e307a8d74712ad2ac36111309262, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/1e6717fb19044bd0822a44639c0955fe, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/76b098206e8e417fb1b119a832ca53b7] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=40.7 K 2024-11-23T15:40:22,963 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/81e8569b11b34244bc3df0fd238d0c6f, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/cdbf86d562a0472d8362d517e72ee77b, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/d336e825e46a45b7b134496887cdcfeb] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=36.0 K 2024-11-23T15:40:22,964 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 81e8569b11b34244bc3df0fd238d0c6f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732376419995 2024-11-23T15:40:22,964 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7324e307a8d74712ad2ac36111309262, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732376419995 2024-11-23T15:40:22,964 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting cdbf86d562a0472d8362d517e72ee77b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732376421293 2024-11-23T15:40:22,964 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e6717fb19044bd0822a44639c0955fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732376421293 2024-11-23T15:40:22,965 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76b098206e8e417fb1b119a832ca53b7, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732376422071 2024-11-23T15:40:22,965 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting d336e825e46a45b7b134496887cdcfeb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732376422071 2024-11-23T15:40:22,979 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#B#compaction#46 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:22,980 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/f9ef2a573bf3462e9ba1b414a5ad6f5a is 50, key is test_row_0/B:col10/1732376422406/Put/seqid=0 2024-11-23T15:40:22,982 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#A#compaction#45 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:22,984 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/e0059d94e89b4b3fbb6ea1ec56b3146a is 50, key is test_row_0/A:col10/1732376422406/Put/seqid=0 2024-11-23T15:40:23,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741883_1059 (size=12663) 2024-11-23T15:40:23,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741882_1058 (size=12663) 2024-11-23T15:40:23,020 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/f9ef2a573bf3462e9ba1b414a5ad6f5a as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/f9ef2a573bf3462e9ba1b414a5ad6f5a 2024-11-23T15:40:23,031 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/B of dad1202300d1ddd3a95c32af1ad98ed0 into f9ef2a573bf3462e9ba1b414a5ad6f5a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:23,031 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:23,031 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/B, priority=13, startTime=1732376422962; duration=0sec 2024-11-23T15:40:23,031 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:23,031 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:B 2024-11-23T15:40:23,031 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:23,033 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:23,034 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/C is initiating minor compaction (all files) 2024-11-23T15:40:23,034 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/C in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:23,034 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/2bb8ca1a852c48b3ba2bc19f8f299d68, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/d432d2037fdb4454acf020761bcd66b6, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/941bb1dd86bb47fbb6a6f5b1764162e5] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=36.0 K 2024-11-23T15:40:23,035 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bb8ca1a852c48b3ba2bc19f8f299d68, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732376419995 2024-11-23T15:40:23,035 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting d432d2037fdb4454acf020761bcd66b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732376421293 2024-11-23T15:40:23,036 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 941bb1dd86bb47fbb6a6f5b1764162e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732376422071 2024-11-23T15:40:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T15:40:23,055 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#C#compaction#47 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:23,056 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/7a120e8159f04093808ab3b75e36338b is 50, key is test_row_0/C:col10/1732376422406/Put/seqid=0 2024-11-23T15:40:23,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-23T15:40:23,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:23,059 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-23T15:40:23,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/a04b046fcb914ac78ecf36032353fe23 is 50, key is test_row_0/A:col10/1732376422445/Put/seqid=0 2024-11-23T15:40:23,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741884_1060 (size=12663) 2024-11-23T15:40:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:23,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:23,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741885_1061 (size=12151) 2024-11-23T15:40:23,084 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/7a120e8159f04093808ab3b75e36338b as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/7a120e8159f04093808ab3b75e36338b 2024-11-23T15:40:23,094 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/C of dad1202300d1ddd3a95c32af1ad98ed0 into 7a120e8159f04093808ab3b75e36338b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:23,095 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:23,096 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/C, priority=13, startTime=1732376422962; duration=0sec 2024-11-23T15:40:23,096 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:23,096 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:C 2024-11-23T15:40:23,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:23,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376483110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:23,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,118 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:23,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376483113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:23,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:23,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:23,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376483113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376483115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:23,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376483117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:23,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:23,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376483224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:23,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376483225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:23,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376483225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:23,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376483225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:23,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:23,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:23,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376483234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:23,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:23,415 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/e0059d94e89b4b3fbb6ea1ec56b3146a as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/e0059d94e89b4b3fbb6ea1ec56b3146a 2024-11-23T15:40:23,426 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/A of dad1202300d1ddd3a95c32af1ad98ed0 into e0059d94e89b4b3fbb6ea1ec56b3146a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:23,426 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:23,426 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/A, priority=13, startTime=1732376422961; duration=0sec 2024-11-23T15:40:23,426 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:23,426 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:A 2024-11-23T15:40:23,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:23,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376483434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:23,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:23,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:23,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376483434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:23,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:23,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376483444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376483444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:23,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376483444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:23,480 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/a04b046fcb914ac78ecf36032353fe23 2024-11-23T15:40:23,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/287df1bce8d947699350d176354971dd is 50, key is test_row_0/B:col10/1732376422445/Put/seqid=0 2024-11-23T15:40:23,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741886_1062 (size=12151) 2024-11-23T15:40:23,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T15:40:23,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:23,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376483745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376483745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:23,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:23,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,748 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:23,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376483754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:23,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:23,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376483755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376483754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:23,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:23,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:23,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:23,897 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/287df1bce8d947699350d176354971dd 2024-11-23T15:40:23,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/a32f23da560f486e9e000260108fdd9d is 50, key is test_row_0/C:col10/1732376422445/Put/seqid=0 2024-11-23T15:40:23,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741887_1063 (size=12151) 2024-11-23T15:40:23,922 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/a32f23da560f486e9e000260108fdd9d 2024-11-23T15:40:23,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/a04b046fcb914ac78ecf36032353fe23 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/a04b046fcb914ac78ecf36032353fe23 2024-11-23T15:40:23,936 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/a04b046fcb914ac78ecf36032353fe23, entries=150, sequenceid=233, filesize=11.9 K 2024-11-23T15:40:23,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/287df1bce8d947699350d176354971dd as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/287df1bce8d947699350d176354971dd 2024-11-23T15:40:23,944 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/287df1bce8d947699350d176354971dd, entries=150, sequenceid=233, filesize=11.9 K 2024-11-23T15:40:23,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/a32f23da560f486e9e000260108fdd9d as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/a32f23da560f486e9e000260108fdd9d 2024-11-23T15:40:23,964 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/a32f23da560f486e9e000260108fdd9d, entries=150, sequenceid=233, filesize=11.9 K 2024-11-23T15:40:23,965 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for dad1202300d1ddd3a95c32af1ad98ed0 in 906ms, sequenceid=233, compaction requested=false 2024-11-23T15:40:23,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:23,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:23,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-23T15:40:23,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-23T15:40:23,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-23T15:40:23,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5350 sec 2024-11-23T15:40:23,975 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.5450 sec 2024-11-23T15:40:24,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:24,257 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:40:24,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-23T15:40:24,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/946f4e52b9da453492915094f8030925 is 50, key is test_row_0/A:col10/1732376423114/Put/seqid=0 2024-11-23T15:40:24,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:24,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376484281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376484283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:24,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:24,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,288 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376484284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:24,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:24,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376484286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:24,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:24,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:24,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376484288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:24,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741888_1064 (size=12151) 2024-11-23T15:40:24,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/946f4e52b9da453492915094f8030925 2024-11-23T15:40:24,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/1b2f75fb3eb94f728a116fac245908e0 is 50, key is test_row_0/B:col10/1732376423114/Put/seqid=0 2024-11-23T15:40:24,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741889_1065 (size=12151) 2024-11-23T15:40:24,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:24,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376484394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:24,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376484394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:24,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376484394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:24,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:24,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376484394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:24,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:24,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376484405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:24,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T15:40:24,575 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:40:24,576 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:40:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-23T15:40:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T15:40:24,579 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:40:24,580 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:40:24,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:40:24,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:24,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376484604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376484604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:24,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376484605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:24,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376484605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:24,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:24,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:24,611 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:24,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:24,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376484624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:24,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T15:40:24,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-23T15:40:24,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:24,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:24,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:24,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:24,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:24,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/1b2f75fb3eb94f728a116fac245908e0 2024-11-23T15:40:24,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/64bc2bb373ba42d085e16860e401c63d is 50, key is test_row_0/C:col10/1732376423114/Put/seqid=0 2024-11-23T15:40:24,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741890_1066 (size=12151) 2024-11-23T15:40:24,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-23T15:40:24,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:24,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:24,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:24,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:24,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:24,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T15:40:24,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:24,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376484914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376484914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376484914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:24,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376484924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:24,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376484944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:24,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:24,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:25,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-23T15:40:25,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:25,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:25,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:25,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:25,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:25,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/64bc2bb373ba42d085e16860e401c63d 2024-11-23T15:40:25,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/946f4e52b9da453492915094f8030925 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/946f4e52b9da453492915094f8030925 2024-11-23T15:40:25,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/946f4e52b9da453492915094f8030925, entries=150, sequenceid=255, filesize=11.9 K 2024-11-23T15:40:25,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-23T15:40:25,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:25,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/1b2f75fb3eb94f728a116fac245908e0 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1b2f75fb3eb94f728a116fac245908e0 2024-11-23T15:40:25,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:25,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:25,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:25,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:25,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T15:40:25,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1b2f75fb3eb94f728a116fac245908e0, entries=150, sequenceid=255, filesize=11.9 K 2024-11-23T15:40:25,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/64bc2bb373ba42d085e16860e401c63d as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/64bc2bb373ba42d085e16860e401c63d 2024-11-23T15:40:25,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/64bc2bb373ba42d085e16860e401c63d, entries=150, sequenceid=255, filesize=11.9 K 2024-11-23T15:40:25,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for dad1202300d1ddd3a95c32af1ad98ed0 in 970ms, sequenceid=255, compaction requested=true 2024-11-23T15:40:25,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:25,227 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:25,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:40:25,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:25,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:40:25,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:25,228 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:25,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:40:25,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:25,229 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:25,229 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/A is initiating minor compaction (all files) 2024-11-23T15:40:25,229 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/A in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:25,229 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/e0059d94e89b4b3fbb6ea1ec56b3146a, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/a04b046fcb914ac78ecf36032353fe23, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/946f4e52b9da453492915094f8030925] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=36.1 K 2024-11-23T15:40:25,230 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:25,230 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0059d94e89b4b3fbb6ea1ec56b3146a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732376422071 2024-11-23T15:40:25,230 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/B is initiating minor compaction (all files) 2024-11-23T15:40:25,230 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/B in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:25,230 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/f9ef2a573bf3462e9ba1b414a5ad6f5a, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/287df1bce8d947699350d176354971dd, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1b2f75fb3eb94f728a116fac245908e0] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=36.1 K 2024-11-23T15:40:25,230 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting a04b046fcb914ac78ecf36032353fe23, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732376422440 2024-11-23T15:40:25,231 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting f9ef2a573bf3462e9ba1b414a5ad6f5a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732376422071 2024-11-23T15:40:25,231 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 287df1bce8d947699350d176354971dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732376422440 2024-11-23T15:40:25,231 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 946f4e52b9da453492915094f8030925, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732376423114 2024-11-23T15:40:25,232 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b2f75fb3eb94f728a116fac245908e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732376423114 2024-11-23T15:40:25,257 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#B#compaction#54 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:25,257 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/3b7d1cf802744cd09518b81758d506e0 is 50, key is test_row_0/B:col10/1732376423114/Put/seqid=0 2024-11-23T15:40:25,269 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#A#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:25,269 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/93516305105f4f3fa01f7cc73b15c787 is 50, key is test_row_0/A:col10/1732376423114/Put/seqid=0 2024-11-23T15:40:25,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741891_1067 (size=12765) 2024-11-23T15:40:25,292 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/3b7d1cf802744cd09518b81758d506e0 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3b7d1cf802744cd09518b81758d506e0 2024-11-23T15:40:25,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741892_1068 (size=12765) 2024-11-23T15:40:25,302 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/B of dad1202300d1ddd3a95c32af1ad98ed0 into 3b7d1cf802744cd09518b81758d506e0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:25,303 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:25,303 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/B, priority=13, startTime=1732376425228; duration=0sec 2024-11-23T15:40:25,303 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:25,303 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:B 2024-11-23T15:40:25,303 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:25,311 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:25,311 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/C is initiating minor compaction (all files) 2024-11-23T15:40:25,311 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/C in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:25,311 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/7a120e8159f04093808ab3b75e36338b, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/a32f23da560f486e9e000260108fdd9d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/64bc2bb373ba42d085e16860e401c63d] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=36.1 K 2024-11-23T15:40:25,312 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a120e8159f04093808ab3b75e36338b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732376422071 2024-11-23T15:40:25,312 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting a32f23da560f486e9e000260108fdd9d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732376422440 2024-11-23T15:40:25,313 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 64bc2bb373ba42d085e16860e401c63d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732376423114 2024-11-23T15:40:25,313 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/93516305105f4f3fa01f7cc73b15c787 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/93516305105f4f3fa01f7cc73b15c787 2024-11-23T15:40:25,329 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/A of dad1202300d1ddd3a95c32af1ad98ed0 into 93516305105f4f3fa01f7cc73b15c787(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:25,329 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:25,329 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/A, priority=13, startTime=1732376425227; duration=0sec 2024-11-23T15:40:25,329 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:25,329 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:A 2024-11-23T15:40:25,341 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#C#compaction#56 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:25,341 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/a5dceffff1f0431f8432459c01d7e7a8 is 50, key is test_row_0/C:col10/1732376423114/Put/seqid=0 2024-11-23T15:40:25,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-23T15:40:25,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:25,351 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-23T15:40:25,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/44ee99398d0447848ccdd25e824782fa is 50, key is test_row_0/A:col10/1732376424270/Put/seqid=0 2024-11-23T15:40:25,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741893_1069 (size=12765) 2024-11-23T15:40:25,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741894_1070 (size=12301) 2024-11-23T15:40:25,373 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/44ee99398d0447848ccdd25e824782fa 2024-11-23T15:40:25,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/fd5e69ec002a4e73b1620dd52a87e2df is 50, key is test_row_0/B:col10/1732376424270/Put/seqid=0 2024-11-23T15:40:25,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741895_1071 (size=12301) 2024-11-23T15:40:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:25,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:25,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376485454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:25,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376485455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:25,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:25,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376485456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:25,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376485457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:25,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:25,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376485458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:25,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:25,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376485564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:25,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376485565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376485565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:25,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376485574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T15:40:25,776 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/a5dceffff1f0431f8432459c01d7e7a8 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/a5dceffff1f0431f8432459c01d7e7a8 2024-11-23T15:40:25,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376485775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:25,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376485775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:25,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:25,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376485775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:25,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:25,791 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/fd5e69ec002a4e73b1620dd52a87e2df 2024-11-23T15:40:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376485785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:25,791 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/C of dad1202300d1ddd3a95c32af1ad98ed0 into a5dceffff1f0431f8432459c01d7e7a8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:25,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,791 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:25,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:25,791 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/C, priority=13, startTime=1732376425228; duration=0sec 2024-11-23T15:40:25,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:25,792 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:25,792 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:C 2024-11-23T15:40:25,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/15a8f9a2c23249cb8e73eeb2f6e27006 is 50, key is test_row_0/C:col10/1732376424270/Put/seqid=0 2024-11-23T15:40:25,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741896_1072 (size=12301) 2024-11-23T15:40:25,868 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/15a8f9a2c23249cb8e73eeb2f6e27006 2024-11-23T15:40:25,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/44ee99398d0447848ccdd25e824782fa as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/44ee99398d0447848ccdd25e824782fa 2024-11-23T15:40:25,884 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/44ee99398d0447848ccdd25e824782fa, entries=150, sequenceid=273, filesize=12.0 K 2024-11-23T15:40:25,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/fd5e69ec002a4e73b1620dd52a87e2df as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fd5e69ec002a4e73b1620dd52a87e2df 2024-11-23T15:40:25,893 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fd5e69ec002a4e73b1620dd52a87e2df, entries=150, sequenceid=273, filesize=12.0 K 2024-11-23T15:40:25,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/15a8f9a2c23249cb8e73eeb2f6e27006 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/15a8f9a2c23249cb8e73eeb2f6e27006 2024-11-23T15:40:25,918 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/15a8f9a2c23249cb8e73eeb2f6e27006, entries=150, sequenceid=273, filesize=12.0 K 2024-11-23T15:40:25,920 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for dad1202300d1ddd3a95c32af1ad98ed0 in 569ms, sequenceid=273, compaction requested=false 2024-11-23T15:40:25,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:25,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:25,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-23T15:40:25,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-23T15:40:25,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-23T15:40:25,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3430 sec 2024-11-23T15:40:25,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.3530 sec 2024-11-23T15:40:26,089 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:40:26,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-23T15:40:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:26,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/74b5df2b752d42e4b72345b34139cfc8 is 50, key is test_row_0/A:col10/1732376425452/Put/seqid=0 2024-11-23T15:40:26,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376486130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:26,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:26,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376486133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:26,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:26,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376486141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:26,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741897_1073 (size=12301) 2024-11-23T15:40:26,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376486152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:26,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376486244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:26,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:26,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376486254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:26,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:26,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:26,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376486264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376486264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:26,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:26,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:26,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376486455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:26,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:26,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376486464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:26,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:26,478 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_1' in dad1202300d1ddd3a95c32af1ad98ed0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:40:26,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376486464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:26,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:26,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376486475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376486477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:26,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:26,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/74b5df2b752d42e4b72345b34139cfc8 2024-11-23T15:40:26,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/130a75da0a844b00886697ee514269f8 is 50, key is test_row_0/B:col10/1732376425452/Put/seqid=0 2024-11-23T15:40:26,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741898_1074 (size=12301) 2024-11-23T15:40:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T15:40:26,735 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:40:26,736 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:40:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-23T15:40:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-23T15:40:26,741 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:40:26,743 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:40:26,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:40:26,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376486764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:26,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:26,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:26,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:26,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376486795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:26,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376486795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376486795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:26,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-23T15:40:26,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-23T15:40:26,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:26,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:26,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:26,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:26,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:27,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/130a75da0a844b00886697ee514269f8 2024-11-23T15:40:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-23T15:40:27,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/517dd48ad1ff4f96928e455d423c125e is 50, key is test_row_0/C:col10/1732376425452/Put/seqid=0 2024-11-23T15:40:27,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-23T15:40:27,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:27,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:27,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:27,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:27,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:27,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741899_1075 (size=12301) 2024-11-23T15:40:27,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-23T15:40:27,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:27,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:27,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:27,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:27,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:27,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376487285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:27,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:27,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:27,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:27,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:27,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376487314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376487314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:27,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376487314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:27,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:27,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:27,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:27,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-23T15:40:27,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-23T15:40:27,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:27,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:27,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:27,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:27,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:27,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/517dd48ad1ff4f96928e455d423c125e 2024-11-23T15:40:27,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/74b5df2b752d42e4b72345b34139cfc8 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/74b5df2b752d42e4b72345b34139cfc8 2024-11-23T15:40:27,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/74b5df2b752d42e4b72345b34139cfc8, entries=150, sequenceid=295, filesize=12.0 K 2024-11-23T15:40:27,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/130a75da0a844b00886697ee514269f8 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/130a75da0a844b00886697ee514269f8 2024-11-23T15:40:27,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/130a75da0a844b00886697ee514269f8, entries=150, sequenceid=295, filesize=12.0 K 2024-11-23T15:40:27,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/517dd48ad1ff4f96928e455d423c125e as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/517dd48ad1ff4f96928e455d423c125e 2024-11-23T15:40:27,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/517dd48ad1ff4f96928e455d423c125e, entries=150, sequenceid=295, filesize=12.0 K 2024-11-23T15:40:27,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for dad1202300d1ddd3a95c32af1ad98ed0 in 1433ms, sequenceid=295, compaction requested=true 2024-11-23T15:40:27,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:27,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:40:27,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:27,522 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:27,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:40:27,522 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:27,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:27,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:40:27,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:27,524 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:27,524 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/A is initiating minor compaction (all files) 2024-11-23T15:40:27,524 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/A in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:27,524 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/93516305105f4f3fa01f7cc73b15c787, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/44ee99398d0447848ccdd25e824782fa, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/74b5df2b752d42e4b72345b34139cfc8] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=36.5 K 2024-11-23T15:40:27,525 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93516305105f4f3fa01f7cc73b15c787, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732376423114 2024-11-23T15:40:27,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-23T15:40:27,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:27,525 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:27,525 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-23T15:40:27,526 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/B is initiating minor compaction (all files) 2024-11-23T15:40:27,526 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/B in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:27,526 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3b7d1cf802744cd09518b81758d506e0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fd5e69ec002a4e73b1620dd52a87e2df, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/130a75da0a844b00886697ee514269f8] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=36.5 K 2024-11-23T15:40:27,526 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44ee99398d0447848ccdd25e824782fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732376424270 2024-11-23T15:40:27,527 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b7d1cf802744cd09518b81758d506e0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732376423114 2024-11-23T15:40:27,527 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74b5df2b752d42e4b72345b34139cfc8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732376425452 2024-11-23T15:40:27,527 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting fd5e69ec002a4e73b1620dd52a87e2df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732376424270 2024-11-23T15:40:27,528 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 130a75da0a844b00886697ee514269f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732376425452 2024-11-23T15:40:27,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/edfc944151564e11ac24bb48193baace is 50, key is test_row_0/A:col10/1732376426140/Put/seqid=0 2024-11-23T15:40:27,556 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#A#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:27,557 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/039cdb3fdb174f9896c4b2ccf8b6762e is 50, key is test_row_0/A:col10/1732376425452/Put/seqid=0 2024-11-23T15:40:27,557 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#B#compaction#65 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:27,557 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/1a32ac83432b4f449a7c38e6b24f3aae is 50, key is test_row_0/B:col10/1732376425452/Put/seqid=0 2024-11-23T15:40:27,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741900_1076 (size=12301) 2024-11-23T15:40:27,571 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/edfc944151564e11ac24bb48193baace 2024-11-23T15:40:27,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741902_1078 (size=13017) 2024-11-23T15:40:27,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741901_1077 (size=13017) 2024-11-23T15:40:27,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/3ad8f7e42cc544428c007c824416e325 is 50, key is test_row_0/B:col10/1732376426140/Put/seqid=0 2024-11-23T15:40:27,601 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/039cdb3fdb174f9896c4b2ccf8b6762e as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/039cdb3fdb174f9896c4b2ccf8b6762e 2024-11-23T15:40:27,601 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/1a32ac83432b4f449a7c38e6b24f3aae as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1a32ac83432b4f449a7c38e6b24f3aae 2024-11-23T15:40:27,613 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/A of dad1202300d1ddd3a95c32af1ad98ed0 into 039cdb3fdb174f9896c4b2ccf8b6762e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:27,613 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:27,613 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/A, priority=13, startTime=1732376427522; duration=0sec 2024-11-23T15:40:27,613 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:27,614 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:A 2024-11-23T15:40:27,614 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:27,615 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:27,615 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/C is initiating minor compaction (all files) 2024-11-23T15:40:27,616 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/C in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:27,616 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/a5dceffff1f0431f8432459c01d7e7a8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/15a8f9a2c23249cb8e73eeb2f6e27006, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/517dd48ad1ff4f96928e455d423c125e] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=36.5 K 2024-11-23T15:40:27,616 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5dceffff1f0431f8432459c01d7e7a8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732376423114 2024-11-23T15:40:27,617 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15a8f9a2c23249cb8e73eeb2f6e27006, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732376424270 2024-11-23T15:40:27,617 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 517dd48ad1ff4f96928e455d423c125e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732376425452 2024-11-23T15:40:27,619 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/B of dad1202300d1ddd3a95c32af1ad98ed0 into 1a32ac83432b4f449a7c38e6b24f3aae(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:27,619 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:27,619 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/B, priority=13, startTime=1732376427522; duration=0sec 2024-11-23T15:40:27,619 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:27,619 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:B 2024-11-23T15:40:27,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741903_1079 (size=12301) 2024-11-23T15:40:27,629 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/3ad8f7e42cc544428c007c824416e325 2024-11-23T15:40:27,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/0e70c6a25b3b48b28e33a79ac7b6a025 is 50, key is test_row_0/C:col10/1732376426140/Put/seqid=0 2024-11-23T15:40:27,641 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#C#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:27,641 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/0d7f8c6a2c5d48889c45794555f74258 is 50, key is test_row_0/C:col10/1732376425452/Put/seqid=0 2024-11-23T15:40:27,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741904_1080 (size=12301) 2024-11-23T15:40:27,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741905_1081 (size=13017) 2024-11-23T15:40:27,708 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/0d7f8c6a2c5d48889c45794555f74258 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/0d7f8c6a2c5d48889c45794555f74258 2024-11-23T15:40:27,720 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/C of dad1202300d1ddd3a95c32af1ad98ed0 into 0d7f8c6a2c5d48889c45794555f74258(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:27,720 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:27,720 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/C, priority=13, startTime=1732376427522; duration=0sec 2024-11-23T15:40:27,720 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:27,720 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:C 2024-11-23T15:40:27,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-23T15:40:28,089 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/0e70c6a25b3b48b28e33a79ac7b6a025 2024-11-23T15:40:28,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/edfc944151564e11ac24bb48193baace as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/edfc944151564e11ac24bb48193baace 2024-11-23T15:40:28,120 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/edfc944151564e11ac24bb48193baace, entries=150, sequenceid=311, filesize=12.0 K 2024-11-23T15:40:28,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/3ad8f7e42cc544428c007c824416e325 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3ad8f7e42cc544428c007c824416e325 2024-11-23T15:40:28,137 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3ad8f7e42cc544428c007c824416e325, entries=150, sequenceid=311, filesize=12.0 K 2024-11-23T15:40:28,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/0e70c6a25b3b48b28e33a79ac7b6a025 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/0e70c6a25b3b48b28e33a79ac7b6a025 2024-11-23T15:40:28,156 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/0e70c6a25b3b48b28e33a79ac7b6a025, entries=150, sequenceid=311, filesize=12.0 K 2024-11-23T15:40:28,157 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=0 B/0 for dad1202300d1ddd3a95c32af1ad98ed0 in 632ms, sequenceid=311, compaction requested=false 2024-11-23T15:40:28,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:28,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:28,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-23T15:40:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-23T15:40:28,162 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-23T15:40:28,163 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4170 sec 2024-11-23T15:40:28,166 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.4280 sec 2024-11-23T15:40:28,347 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:40:28,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T15:40:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:28,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/de6201e4adc740ffa7962fd42b0e57c8 is 50, key is test_row_0/A:col10/1732376428339/Put/seqid=0 2024-11-23T15:40:28,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741906_1082 (size=22065) 2024-11-23T15:40:28,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/de6201e4adc740ffa7962fd42b0e57c8 2024-11-23T15:40:28,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:28,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376488432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376488432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:28,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376488442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:28,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:28,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/984b38124ed745caa0f7903e8371efac is 50, key is test_row_0/B:col10/1732376428339/Put/seqid=0 2024-11-23T15:40:28,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376488444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:28,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741907_1083 (size=12301) 2024-11-23T15:40:28,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/984b38124ed745caa0f7903e8371efac 2024-11-23T15:40:28,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376488489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:28,495 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_1' in dad1202300d1ddd3a95c32af1ad98ed0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:40:28,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:28,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/a85d0160095e49509f93acbd52fbaaf0 is 50, key is test_row_0/C:col10/1732376428339/Put/seqid=0 2024-11-23T15:40:28,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741908_1084 (size=12301) 2024-11-23T15:40:28,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/a85d0160095e49509f93acbd52fbaaf0 2024-11-23T15:40:28,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/de6201e4adc740ffa7962fd42b0e57c8 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/de6201e4adc740ffa7962fd42b0e57c8 2024-11-23T15:40:28,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/de6201e4adc740ffa7962fd42b0e57c8, entries=350, sequenceid=325, filesize=21.5 K 2024-11-23T15:40:28,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/984b38124ed745caa0f7903e8371efac as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/984b38124ed745caa0f7903e8371efac 2024-11-23T15:40:28,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376488555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:28,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:28,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/984b38124ed745caa0f7903e8371efac, entries=150, sequenceid=325, filesize=12.0 K 2024-11-23T15:40:28,556 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376488555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376488555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:28,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:28,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/a85d0160095e49509f93acbd52fbaaf0 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/a85d0160095e49509f93acbd52fbaaf0 2024-11-23T15:40:28,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/a85d0160095e49509f93acbd52fbaaf0, entries=150, sequenceid=325, filesize=12.0 K 2024-11-23T15:40:28,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for dad1202300d1ddd3a95c32af1ad98ed0 in 219ms, sequenceid=325, compaction requested=true 2024-11-23T15:40:28,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:28,566 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:28,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:40:28,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:28,567 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:28,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:40:28,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:28,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:40:28,568 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47383 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:28,568 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:28,568 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/A is initiating minor compaction (all files) 2024-11-23T15:40:28,568 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/B is initiating minor compaction (all files) 2024-11-23T15:40:28,568 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/A in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:28,568 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/B in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:28,568 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/039cdb3fdb174f9896c4b2ccf8b6762e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/edfc944151564e11ac24bb48193baace, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/de6201e4adc740ffa7962fd42b0e57c8] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=46.3 K 2024-11-23T15:40:28,568 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1a32ac83432b4f449a7c38e6b24f3aae, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3ad8f7e42cc544428c007c824416e325, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/984b38124ed745caa0f7903e8371efac] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=36.7 K 2024-11-23T15:40:28,569 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a32ac83432b4f449a7c38e6b24f3aae, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732376425452 2024-11-23T15:40:28,569 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 039cdb3fdb174f9896c4b2ccf8b6762e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732376425452 2024-11-23T15:40:28,569 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting edfc944151564e11ac24bb48193baace, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732376426111 2024-11-23T15:40:28,569 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ad8f7e42cc544428c007c824416e325, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732376426111 2024-11-23T15:40:28,570 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting de6201e4adc740ffa7962fd42b0e57c8, keycount=350, bloomtype=ROW, size=21.5 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732376428308 2024-11-23T15:40:28,570 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 984b38124ed745caa0f7903e8371efac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732376428339 2024-11-23T15:40:28,570 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:28,572 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:40:28,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T15:40:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:28,585 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#B#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:28,585 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#A#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:28,586 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/6b0f22d43fe347c98ecf74623d108bdc is 50, key is test_row_0/B:col10/1732376428339/Put/seqid=0 2024-11-23T15:40:28,586 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/0c4597e47bbd4918b328f88a7543cede is 50, key is test_row_0/A:col10/1732376428339/Put/seqid=0 2024-11-23T15:40:28,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/0013c0b76e824aaf9df9ed9f210b18ee is 50, key is test_row_0/A:col10/1732376428431/Put/seqid=0 2024-11-23T15:40:28,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741909_1085 (size=13119) 2024-11-23T15:40:28,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376488622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:28,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:28,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741911_1087 (size=13119) 2024-11-23T15:40:28,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741910_1086 (size=12301) 2024-11-23T15:40:28,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376488734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:28,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:28,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376488764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:28,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:28,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376488764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:28,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:28,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376488765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:28,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-23T15:40:28,887 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:40:28,888 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:40:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-23T15:40:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T15:40:28,891 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:40:28,892 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:40:28,892 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:40:28,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:28,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376488955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:28,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:28,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T15:40:29,013 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/0c4597e47bbd4918b328f88a7543cede as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/0c4597e47bbd4918b328f88a7543cede 2024-11-23T15:40:29,020 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/A of dad1202300d1ddd3a95c32af1ad98ed0 into 0c4597e47bbd4918b328f88a7543cede(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:29,020 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:29,020 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/A, priority=13, startTime=1732376428566; duration=0sec 2024-11-23T15:40:29,020 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:29,020 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:A 2024-11-23T15:40:29,020 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:29,022 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:29,023 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/C is initiating minor compaction (all files) 2024-11-23T15:40:29,023 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/C in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:29,023 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/0d7f8c6a2c5d48889c45794555f74258, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/0e70c6a25b3b48b28e33a79ac7b6a025, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/a85d0160095e49509f93acbd52fbaaf0] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=36.7 K 2024-11-23T15:40:29,024 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d7f8c6a2c5d48889c45794555f74258, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732376425452 2024-11-23T15:40:29,024 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e70c6a25b3b48b28e33a79ac7b6a025, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1732376426111 2024-11-23T15:40:29,024 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting a85d0160095e49509f93acbd52fbaaf0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732376428339 2024-11-23T15:40:29,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/0013c0b76e824aaf9df9ed9f210b18ee 2024-11-23T15:40:29,044 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/6b0f22d43fe347c98ecf74623d108bdc as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/6b0f22d43fe347c98ecf74623d108bdc 2024-11-23T15:40:29,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-23T15:40:29,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:29,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:29,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:29,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:29,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:29,052 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#C#compaction#75 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:29,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/cebf219fbd21477b85624fc0c0641e58 is 50, key is test_row_0/B:col10/1732376428431/Put/seqid=0 2024-11-23T15:40:29,053 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/df85719833b94f7b93916fb1e64924c0 is 50, key is test_row_0/C:col10/1732376428339/Put/seqid=0 2024-11-23T15:40:29,058 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/B of dad1202300d1ddd3a95c32af1ad98ed0 into 6b0f22d43fe347c98ecf74623d108bdc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:29,058 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:29,059 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/B, priority=13, startTime=1732376428567; duration=0sec 2024-11-23T15:40:29,059 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:29,059 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:B 2024-11-23T15:40:29,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376489075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376489075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:29,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376489075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:29,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741913_1089 (size=13119) 2024-11-23T15:40:29,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741912_1088 (size=12301) 2024-11-23T15:40:29,119 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/df85719833b94f7b93916fb1e64924c0 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/df85719833b94f7b93916fb1e64924c0 2024-11-23T15:40:29,131 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/C of dad1202300d1ddd3a95c32af1ad98ed0 into df85719833b94f7b93916fb1e64924c0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:29,131 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:29,131 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/C, priority=13, startTime=1732376428567; duration=0sec 2024-11-23T15:40:29,131 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:29,131 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:C 2024-11-23T15:40:29,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-23T15:40:29,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:29,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:29,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:29,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:29,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T15:40:29,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376489266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:29,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-23T15:40:29,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:29,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:29,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:29,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:29,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:29,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/cebf219fbd21477b85624fc0c0641e58 2024-11-23T15:40:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T15:40:29,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-23T15:40:29,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:29,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:29,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:29,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:29,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:29,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/58a726bac7b9457c9665c3e01bf83b23 is 50, key is test_row_0/C:col10/1732376428431/Put/seqid=0 2024-11-23T15:40:29,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741914_1090 (size=12301) 2024-11-23T15:40:29,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/58a726bac7b9457c9665c3e01bf83b23 2024-11-23T15:40:29,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/0013c0b76e824aaf9df9ed9f210b18ee as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/0013c0b76e824aaf9df9ed9f210b18ee 2024-11-23T15:40:29,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/0013c0b76e824aaf9df9ed9f210b18ee, entries=150, sequenceid=351, filesize=12.0 K 2024-11-23T15:40:29,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/cebf219fbd21477b85624fc0c0641e58 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/cebf219fbd21477b85624fc0c0641e58 2024-11-23T15:40:29,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/cebf219fbd21477b85624fc0c0641e58, entries=150, sequenceid=351, filesize=12.0 K 2024-11-23T15:40:29,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/58a726bac7b9457c9665c3e01bf83b23 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/58a726bac7b9457c9665c3e01bf83b23 2024-11-23T15:40:29,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/58a726bac7b9457c9665c3e01bf83b23, entries=150, sequenceid=351, filesize=12.0 K 2024-11-23T15:40:29,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for dad1202300d1ddd3a95c32af1ad98ed0 in 995ms, sequenceid=351, compaction requested=false 2024-11-23T15:40:29,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:29,585 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:40:29,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T15:40:29,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/be645cafb7044cd6885af4508d3e7752 is 50, key is test_row_0/A:col10/1732376429584/Put/seqid=0 2024-11-23T15:40:29,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741915_1091 (size=14741) 2024-11-23T15:40:29,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376489621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:29,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:29,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376489622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:29,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:29,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376489623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:29,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:29,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-23T15:40:29,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:29,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:29,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:29,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:29,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:29,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:29,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376489734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:29,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:29,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376489734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:29,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:29,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:29,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376489734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:29,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:29,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:29,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376489774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:29,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:29,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-23T15:40:29,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:29,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:29,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:29,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:29,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:29,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:29,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376489944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:29,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:29,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376489945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:29,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:29,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376489947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:29,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:29,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-23T15:40:29,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:29,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:29,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:29,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:29,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:29,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:29,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/be645cafb7044cd6885af4508d3e7752 2024-11-23T15:40:30,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/5afafbe8689143338b3e699f4bc3b0d2 is 50, key is test_row_0/B:col10/1732376429584/Put/seqid=0 2024-11-23T15:40:30,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741916_1092 (size=12301) 2024-11-23T15:40:30,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T15:40:30,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/5afafbe8689143338b3e699f4bc3b0d2 2024-11-23T15:40:30,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/61ae252469fe45ed970d7a60b8dd832c is 50, key is test_row_0/C:col10/1732376429584/Put/seqid=0 2024-11-23T15:40:30,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741917_1093 (size=12301) 2024-11-23T15:40:30,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/61ae252469fe45ed970d7a60b8dd832c 2024-11-23T15:40:30,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/be645cafb7044cd6885af4508d3e7752 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/be645cafb7044cd6885af4508d3e7752 2024-11-23T15:40:30,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/be645cafb7044cd6885af4508d3e7752, entries=200, sequenceid=365, filesize=14.4 K 2024-11-23T15:40:30,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/5afafbe8689143338b3e699f4bc3b0d2 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/5afafbe8689143338b3e699f4bc3b0d2 2024-11-23T15:40:30,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/5afafbe8689143338b3e699f4bc3b0d2, entries=150, sequenceid=365, filesize=12.0 K 2024-11-23T15:40:30,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/61ae252469fe45ed970d7a60b8dd832c as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/61ae252469fe45ed970d7a60b8dd832c 2024-11-23T15:40:30,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/61ae252469fe45ed970d7a60b8dd832c, entries=150, sequenceid=365, filesize=12.0 K 2024-11-23T15:40:30,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for dad1202300d1ddd3a95c32af1ad98ed0 in 532ms, sequenceid=365, compaction requested=true 2024-11-23T15:40:30,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:30,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:40:30,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:30,119 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:30,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:40:30,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:30,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:40:30,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-23T15:40:30,119 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:30,121 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:30,121 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:30,121 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/B is initiating minor compaction (all files) 2024-11-23T15:40:30,121 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/A is initiating minor compaction (all files) 2024-11-23T15:40:30,121 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/A in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:30,121 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/B in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:30,121 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/6b0f22d43fe347c98ecf74623d108bdc, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/cebf219fbd21477b85624fc0c0641e58, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/5afafbe8689143338b3e699f4bc3b0d2] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=36.8 K 2024-11-23T15:40:30,121 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/0c4597e47bbd4918b328f88a7543cede, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/0013c0b76e824aaf9df9ed9f210b18ee, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/be645cafb7044cd6885af4508d3e7752] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=39.2 K 2024-11-23T15:40:30,124 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c4597e47bbd4918b328f88a7543cede, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732376428339 2024-11-23T15:40:30,124 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0013c0b76e824aaf9df9ed9f210b18ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732376428431 2024-11-23T15:40:30,125 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting be645cafb7044cd6885af4508d3e7752, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732376428589 2024-11-23T15:40:30,126 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b0f22d43fe347c98ecf74623d108bdc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732376428339 2024-11-23T15:40:30,127 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting cebf219fbd21477b85624fc0c0641e58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732376428431 2024-11-23T15:40:30,128 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 5afafbe8689143338b3e699f4bc3b0d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732376428589 2024-11-23T15:40:30,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-23T15:40:30,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:30,134 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T15:40:30,138 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#A#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:30,139 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/ff5f981f6db241cd95beb8f4a7c47790 is 50, key is test_row_0/A:col10/1732376429584/Put/seqid=0 2024-11-23T15:40:30,157 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#B#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:30,157 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/1cfa865176de4f8e95ed7a61e215b9a4 is 50, key is test_row_0/B:col10/1732376429584/Put/seqid=0 2024-11-23T15:40:30,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/cb2f94148dae422c8d5934c9986e464c is 50, key is test_row_0/A:col10/1732376429594/Put/seqid=0 2024-11-23T15:40:30,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741918_1094 (size=13221) 2024-11-23T15:40:30,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741919_1095 (size=13221) 2024-11-23T15:40:30,180 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/ff5f981f6db241cd95beb8f4a7c47790 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/ff5f981f6db241cd95beb8f4a7c47790 2024-11-23T15:40:30,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741920_1096 (size=12301) 2024-11-23T15:40:30,194 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/A of dad1202300d1ddd3a95c32af1ad98ed0 into ff5f981f6db241cd95beb8f4a7c47790(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:30,194 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:30,194 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/A, priority=13, startTime=1732376430119; duration=0sec 2024-11-23T15:40:30,194 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:30,194 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:A 2024-11-23T15:40:30,194 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:30,195 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/cb2f94148dae422c8d5934c9986e464c 2024-11-23T15:40:30,198 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:30,199 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/C is initiating minor compaction (all files) 2024-11-23T15:40:30,199 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/C in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:30,199 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/df85719833b94f7b93916fb1e64924c0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/58a726bac7b9457c9665c3e01bf83b23, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/61ae252469fe45ed970d7a60b8dd832c] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=36.8 K 2024-11-23T15:40:30,200 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting df85719833b94f7b93916fb1e64924c0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732376428339 2024-11-23T15:40:30,201 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58a726bac7b9457c9665c3e01bf83b23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732376428431 2024-11-23T15:40:30,202 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61ae252469fe45ed970d7a60b8dd832c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732376428589 2024-11-23T15:40:30,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/c7bcb4729d9f4839b27e0a21fc4e36d9 is 50, key is test_row_0/B:col10/1732376429594/Put/seqid=0 2024-11-23T15:40:30,239 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#C#compaction#85 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:30,240 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/02e8220a8f2c471884977e0132084388 is 50, key is test_row_0/C:col10/1732376429584/Put/seqid=0 2024-11-23T15:40:30,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:30,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:30,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741922_1098 (size=13221) 2024-11-23T15:40:30,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741921_1097 (size=12301) 2024-11-23T15:40:30,290 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/02e8220a8f2c471884977e0132084388 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/02e8220a8f2c471884977e0132084388 2024-11-23T15:40:30,301 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/C of dad1202300d1ddd3a95c32af1ad98ed0 into 02e8220a8f2c471884977e0132084388(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:30,301 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:30,301 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/C, priority=13, startTime=1732376430119; duration=0sec 2024-11-23T15:40:30,301 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:30,301 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:C 2024-11-23T15:40:30,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:30,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376490297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376490297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:30,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,303 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:30,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:30,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:30,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376490303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:30,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:30,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376490414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:30,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:30,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376490414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,416 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:30,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:30,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376490425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:30,586 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/1cfa865176de4f8e95ed7a61e215b9a4 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1cfa865176de4f8e95ed7a61e215b9a4 2024-11-23T15:40:30,595 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/B of dad1202300d1ddd3a95c32af1ad98ed0 into 1cfa865176de4f8e95ed7a61e215b9a4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:30,595 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:30,595 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/B, priority=13, startTime=1732376430119; duration=0sec 2024-11-23T15:40:30,596 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:30,596 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:B 2024-11-23T15:40:30,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376490624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376490624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:30,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376490634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:30,683 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/c7bcb4729d9f4839b27e0a21fc4e36d9 2024-11-23T15:40:30,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/d5b902ef856c4b238403244f0cfab504 is 50, key is test_row_0/C:col10/1732376429594/Put/seqid=0 2024-11-23T15:40:30,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741923_1099 (size=12301) 2024-11-23T15:40:30,721 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/d5b902ef856c4b238403244f0cfab504 2024-11-23T15:40:30,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/cb2f94148dae422c8d5934c9986e464c as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/cb2f94148dae422c8d5934c9986e464c 2024-11-23T15:40:30,738 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/cb2f94148dae422c8d5934c9986e464c, entries=150, sequenceid=390, filesize=12.0 K 2024-11-23T15:40:30,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/c7bcb4729d9f4839b27e0a21fc4e36d9 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/c7bcb4729d9f4839b27e0a21fc4e36d9 2024-11-23T15:40:30,747 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/c7bcb4729d9f4839b27e0a21fc4e36d9, entries=150, sequenceid=390, filesize=12.0 K 2024-11-23T15:40:30,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/d5b902ef856c4b238403244f0cfab504 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/d5b902ef856c4b238403244f0cfab504 2024-11-23T15:40:30,767 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/d5b902ef856c4b238403244f0cfab504, entries=150, sequenceid=390, filesize=12.0 K 2024-11-23T15:40:30,768 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for dad1202300d1ddd3a95c32af1ad98ed0 in 634ms, sequenceid=390, compaction requested=false 2024-11-23T15:40:30,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:30,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:30,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-23T15:40:30,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-23T15:40:30,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-23T15:40:30,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8790 sec 2024-11-23T15:40:30,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.8870 sec 2024-11-23T15:40:30,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:30,787 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:40:30,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T15:40:30,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/a060c98acbc34281af60303ddb9b047c is 50, key is test_row_0/A:col10/1732376430295/Put/seqid=0 2024-11-23T15:40:30,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741924_1100 (size=14741) 2024-11-23T15:40:30,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:30,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376490900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:30,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:30,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376490934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:30,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376490934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:30,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:30,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376490944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:30,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:30,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:31,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:31,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376491015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:31,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T15:40:31,035 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:40:31,036 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:40:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-23T15:40:31,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T15:40:31,039 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:40:31,040 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:40:31,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:40:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T15:40:31,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-23T15:40:31,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:31,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:31,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:31,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:31,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:31,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/a060c98acbc34281af60303ddb9b047c 2024-11-23T15:40:31,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376491224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:31,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,236 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:31,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/fa3f43165b43449fa4a63a00e288b749 is 50, key is test_row_0/B:col10/1732376430295/Put/seqid=0 2024-11-23T15:40:31,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741925_1101 (size=12301) 2024-11-23T15:40:31,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/fa3f43165b43449fa4a63a00e288b749 2024-11-23T15:40:31,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/0c8336167d744cabb237a4dac8b2a376 is 50, key is test_row_0/C:col10/1732376430295/Put/seqid=0 2024-11-23T15:40:31,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-23T15:40:31,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:31,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:31,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:31,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:31,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T15:40:31,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741926_1102 (size=12301) 2024-11-23T15:40:31,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/0c8336167d744cabb237a4dac8b2a376 2024-11-23T15:40:31,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/a060c98acbc34281af60303ddb9b047c as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/a060c98acbc34281af60303ddb9b047c 2024-11-23T15:40:31,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/a060c98acbc34281af60303ddb9b047c, entries=200, sequenceid=405, filesize=14.4 K 2024-11-23T15:40:31,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/fa3f43165b43449fa4a63a00e288b749 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fa3f43165b43449fa4a63a00e288b749 2024-11-23T15:40:31,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fa3f43165b43449fa4a63a00e288b749, entries=150, sequenceid=405, filesize=12.0 K 2024-11-23T15:40:31,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/0c8336167d744cabb237a4dac8b2a376 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/0c8336167d744cabb237a4dac8b2a376 2024-11-23T15:40:31,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/0c8336167d744cabb237a4dac8b2a376, entries=150, sequenceid=405, filesize=12.0 K 2024-11-23T15:40:31,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for dad1202300d1ddd3a95c32af1ad98ed0 in 612ms, sequenceid=405, compaction requested=true 2024-11-23T15:40:31,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:31,400 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:31,402 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:31,402 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/A is initiating minor compaction (all files) 2024-11-23T15:40:31,402 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/A in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:31,402 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/ff5f981f6db241cd95beb8f4a7c47790, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/cb2f94148dae422c8d5934c9986e464c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/a060c98acbc34281af60303ddb9b047c] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=39.3 K 2024-11-23T15:40:31,403 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff5f981f6db241cd95beb8f4a7c47790, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732376428589 2024-11-23T15:40:31,403 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb2f94148dae422c8d5934c9986e464c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732376429594 2024-11-23T15:40:31,404 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting a060c98acbc34281af60303ddb9b047c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732376430293 2024-11-23T15:40:31,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:40:31,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:31,411 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:31,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:40:31,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:31,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:40:31,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:31,415 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#A#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:31,415 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:31,415 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/B is initiating minor compaction (all files) 2024-11-23T15:40:31,415 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/B in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:31,416 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1cfa865176de4f8e95ed7a61e215b9a4, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/c7bcb4729d9f4839b27e0a21fc4e36d9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fa3f43165b43449fa4a63a00e288b749] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=36.9 K 2024-11-23T15:40:31,416 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cfa865176de4f8e95ed7a61e215b9a4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732376428589 2024-11-23T15:40:31,416 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/b7dce56890dd4763a7919e2816206887 is 50, key is test_row_0/A:col10/1732376430295/Put/seqid=0 2024-11-23T15:40:31,417 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting c7bcb4729d9f4839b27e0a21fc4e36d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732376429594 2024-11-23T15:40:31,417 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting fa3f43165b43449fa4a63a00e288b749, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732376430293 2024-11-23T15:40:31,441 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#B#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:31,442 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/3c918ef77a6548548f38bb06aaf649b7 is 50, key is test_row_0/B:col10/1732376430295/Put/seqid=0 2024-11-23T15:40:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:31,461 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:40:31,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T15:40:31,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741927_1103 (size=13323) 2024-11-23T15:40:31,478 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/b7dce56890dd4763a7919e2816206887 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/b7dce56890dd4763a7919e2816206887 2024-11-23T15:40:31,488 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/A of dad1202300d1ddd3a95c32af1ad98ed0 into b7dce56890dd4763a7919e2816206887(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:31,488 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:31,488 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/A, priority=13, startTime=1732376431400; duration=0sec 2024-11-23T15:40:31,488 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:31,489 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:A 2024-11-23T15:40:31,489 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:31,490 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:31,490 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/C is initiating minor compaction (all files) 2024-11-23T15:40:31,490 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/C in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:31,490 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/02e8220a8f2c471884977e0132084388, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/d5b902ef856c4b238403244f0cfab504, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/0c8336167d744cabb237a4dac8b2a376] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=36.9 K 2024-11-23T15:40:31,491 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02e8220a8f2c471884977e0132084388, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732376428589 2024-11-23T15:40:31,491 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5b902ef856c4b238403244f0cfab504, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=390, earliestPutTs=1732376429594 2024-11-23T15:40:31,492 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c8336167d744cabb237a4dac8b2a376, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732376430293 2024-11-23T15:40:31,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/6b1c39a157794e36b0921adc9caff695 is 50, key is test_row_0/A:col10/1732376431460/Put/seqid=0 2024-11-23T15:40:31,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:31,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376491486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:31,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,505 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:31,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-23T15:40:31,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:31,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:31,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:31,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:31,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:31,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:31,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741928_1104 (size=13323) 2024-11-23T15:40:31,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376491504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:31,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:31,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:31,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376491506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:31,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:31,534 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#C#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:31,535 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/ddf1a5219af544499dbac5d8dd7f5e50 is 50, key is test_row_0/C:col10/1732376430295/Put/seqid=0 2024-11-23T15:40:31,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:31,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376491545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:31,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:31,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741929_1105 (size=14741) 2024-11-23T15:40:31,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/6b1c39a157794e36b0921adc9caff695 2024-11-23T15:40:31,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741930_1106 (size=13323) 2024-11-23T15:40:31,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/529b33480c0f447198a03cde03786eb8 is 50, key is test_row_0/B:col10/1732376431460/Put/seqid=0 2024-11-23T15:40:31,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741931_1107 (size=12301) 2024-11-23T15:40:31,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:31,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376491615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:31,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/529b33480c0f447198a03cde03786eb8 2024-11-23T15:40:31,623 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:31,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376491631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:31,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:31,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,637 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376491631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:31,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:31,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/44d0a249fbea40ea97630ccf358763d0 is 50, key is test_row_0/C:col10/1732376431460/Put/seqid=0 2024-11-23T15:40:31,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-23T15:40:31,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:31,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:31,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:31,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:31,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:31,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T15:40:31,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741932_1108 (size=12301) 2024-11-23T15:40:31,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/44d0a249fbea40ea97630ccf358763d0 2024-11-23T15:40:31,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/6b1c39a157794e36b0921adc9caff695 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6b1c39a157794e36b0921adc9caff695 2024-11-23T15:40:31,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6b1c39a157794e36b0921adc9caff695, entries=200, sequenceid=430, filesize=14.4 K 2024-11-23T15:40:31,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/529b33480c0f447198a03cde03786eb8 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/529b33480c0f447198a03cde03786eb8 2024-11-23T15:40:31,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/529b33480c0f447198a03cde03786eb8, entries=150, sequenceid=430, filesize=12.0 K 2024-11-23T15:40:31,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/44d0a249fbea40ea97630ccf358763d0 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/44d0a249fbea40ea97630ccf358763d0 2024-11-23T15:40:31,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/44d0a249fbea40ea97630ccf358763d0, entries=150, sequenceid=430, filesize=12.0 K 2024-11-23T15:40:31,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for dad1202300d1ddd3a95c32af1ad98ed0 in 329ms, sequenceid=430, compaction requested=false 2024-11-23T15:40:31,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:31,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-23T15:40:31,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:31,814 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T15:40:31,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/8091e03607e2442c95de26b7851df432 is 50, key is test_row_0/A:col10/1732376431498/Put/seqid=0 2024-11-23T15:40:31,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741933_1109 (size=12301) 2024-11-23T15:40:31,834 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/8091e03607e2442c95de26b7851df432 2024-11-23T15:40:31,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:31,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:31,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/fe5159febd2c4b73a1d181c12051d1c6 is 50, key is test_row_0/B:col10/1732376431498/Put/seqid=0 2024-11-23T15:40:31,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741934_1110 (size=12301) 2024-11-23T15:40:31,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:31,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376491892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:31,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:31,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376491891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:31,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:31,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376491892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:31,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:31,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:31,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:31,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:31,927 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/3c918ef77a6548548f38bb06aaf649b7 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3c918ef77a6548548f38bb06aaf649b7 2024-11-23T15:40:31,939 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/B of dad1202300d1ddd3a95c32af1ad98ed0 into 3c918ef77a6548548f38bb06aaf649b7(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:31,940 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:31,940 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/B, priority=13, startTime=1732376431411; duration=0sec 2024-11-23T15:40:31,940 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:31,940 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:B 2024-11-23T15:40:32,002 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/ddf1a5219af544499dbac5d8dd7f5e50 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/ddf1a5219af544499dbac5d8dd7f5e50 2024-11-23T15:40:32,010 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/C of dad1202300d1ddd3a95c32af1ad98ed0 into ddf1a5219af544499dbac5d8dd7f5e50(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:32,010 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:32,010 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/C, priority=13, startTime=1732376431411; duration=0sec 2024-11-23T15:40:32,010 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:32,010 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:C 2024-11-23T15:40:32,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:32,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376492005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:32,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:32,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:32,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:32,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:32,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376492005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:32,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:32,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376492005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:32,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:32,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376492065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:32,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:32,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:32,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T15:40:32,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376492215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:32,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:32,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:32,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:32,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:32,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376492215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:32,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:32,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:32,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:32,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376492225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:32,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:32,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:32,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:32,269 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/fe5159febd2c4b73a1d181c12051d1c6 2024-11-23T15:40:32,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/f470a05edc104e35ab787f6588170ab4 is 50, key is test_row_0/C:col10/1732376431498/Put/seqid=0 2024-11-23T15:40:32,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741935_1111 (size=12301) 2024-11-23T15:40:32,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:32,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55444 deadline: 1732376492524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:32,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:32,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376492524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:32,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:32,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:32,528 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_1' in dad1202300d1ddd3a95c32af1ad98ed0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:40:32,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376492525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:32,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:32,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:32,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:32,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:32,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:32,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:32,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:32,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376492534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:32,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:32,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:32,536 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:32,730 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/f470a05edc104e35ab787f6588170ab4 2024-11-23T15:40:32,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/8091e03607e2442c95de26b7851df432 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/8091e03607e2442c95de26b7851df432 2024-11-23T15:40:32,750 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/8091e03607e2442c95de26b7851df432, entries=150, sequenceid=443, filesize=12.0 K 2024-11-23T15:40:32,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/fe5159febd2c4b73a1d181c12051d1c6 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fe5159febd2c4b73a1d181c12051d1c6 2024-11-23T15:40:32,759 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fe5159febd2c4b73a1d181c12051d1c6, entries=150, sequenceid=443, filesize=12.0 K 2024-11-23T15:40:32,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/f470a05edc104e35ab787f6588170ab4 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/f470a05edc104e35ab787f6588170ab4 2024-11-23T15:40:32,765 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/f470a05edc104e35ab787f6588170ab4, entries=150, sequenceid=443, filesize=12.0 K 2024-11-23T15:40:32,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-23T15:40:32,766 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for dad1202300d1ddd3a95c32af1ad98ed0 in 952ms, sequenceid=443, compaction requested=true 2024-11-23T15:40:32,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:32,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:32,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-23T15:40:32,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-23T15:40:32,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-23T15:40:32,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7290 sec 2024-11-23T15:40:32,773 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.7350 sec 2024-11-23T15:40:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:33,037 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:40:33,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T15:40:33,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/26c0f4119ffe473d8b51b4aa52a9800b is 50, key is test_row_0/A:col10/1732376431888/Put/seqid=0 2024-11-23T15:40:33,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376493048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:33,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:33,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376493048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:33,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:33,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:33,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376493049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:33,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,054 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:33,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741936_1112 (size=12301) 2024-11-23T15:40:33,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:33,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376493074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:33,078 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_0' in dad1202300d1ddd3a95c32af1ad98ed0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:40:33,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:33,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:33,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376493164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:33,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376493164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:33,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:33,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376493164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:33,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T15:40:33,184 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:40:33,185 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:40:33,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-23T15:40:33,187 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:40:33,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T15:40:33,188 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:40:33,188 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:40:33,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T15:40:33,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-23T15:40:33,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:33,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:33,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:33,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:33,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:33,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:33,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376493375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:33,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:33,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376493375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:33,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:33,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:33,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376493375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:33,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,379 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:33,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:33,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/26c0f4119ffe473d8b51b4aa52a9800b 2024-11-23T15:40:33,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/c44d23b185564f918e691852be861797 is 50, key is test_row_0/B:col10/1732376431888/Put/seqid=0 2024-11-23T15:40:33,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-23T15:40:33,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:33,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:33,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:33,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:33,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:33,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T15:40:33,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741937_1113 (size=12301) 2024-11-23T15:40:33,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-23T15:40:33,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:33,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:33,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:33,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:33,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:33,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376493684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:33,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:33,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376493685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:33,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376493685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:33,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:33,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:33,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:33,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:33,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-23T15:40:33,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:33,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:33,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:33,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:33,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:33,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T15:40:33,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/c44d23b185564f918e691852be861797 2024-11-23T15:40:33,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/119d66470b4941749c6fe30326140f61 is 50, key is test_row_0/C:col10/1732376431888/Put/seqid=0 2024-11-23T15:40:33,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741938_1114 (size=12301) 2024-11-23T15:40:33,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-23T15:40:33,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:33,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:33,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:33,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:33,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:33,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:34,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-23T15:40:34,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:34,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:34,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:34,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:34,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:34,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:34,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:34,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376494195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:34,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:34,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:34,201 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:34,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:34,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376494204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:34,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:34,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376494204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:34,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:34,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:34,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:34,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:34,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:34,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:34,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-23T15:40:34,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:34,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:34,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:34,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:34,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:34,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:34,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T15:40:34,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/119d66470b4941749c6fe30326140f61 2024-11-23T15:40:34,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/26c0f4119ffe473d8b51b4aa52a9800b as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/26c0f4119ffe473d8b51b4aa52a9800b 2024-11-23T15:40:34,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/26c0f4119ffe473d8b51b4aa52a9800b, entries=150, sequenceid=471, filesize=12.0 K 2024-11-23T15:40:34,387 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/c44d23b185564f918e691852be861797 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/c44d23b185564f918e691852be861797 2024-11-23T15:40:34,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/c44d23b185564f918e691852be861797, entries=150, sequenceid=471, filesize=12.0 K 2024-11-23T15:40:34,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/119d66470b4941749c6fe30326140f61 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/119d66470b4941749c6fe30326140f61 2024-11-23T15:40:34,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/119d66470b4941749c6fe30326140f61, entries=150, sequenceid=471, filesize=12.0 K 2024-11-23T15:40:34,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for dad1202300d1ddd3a95c32af1ad98ed0 in 1368ms, sequenceid=471, compaction requested=true 2024-11-23T15:40:34,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:34,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:40:34,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:34,405 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:40:34,405 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:40:34,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:40:34,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:34,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:40:34,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:34,409 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52666 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:40:34,409 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/A is initiating minor compaction (all files) 2024-11-23T15:40:34,409 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/A in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:34,409 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/b7dce56890dd4763a7919e2816206887, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6b1c39a157794e36b0921adc9caff695, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/8091e03607e2442c95de26b7851df432, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/26c0f4119ffe473d8b51b4aa52a9800b] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=51.4 K 2024-11-23T15:40:34,409 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:40:34,409 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/B is initiating minor compaction (all files) 2024-11-23T15:40:34,410 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/B in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:34,410 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3c918ef77a6548548f38bb06aaf649b7, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/529b33480c0f447198a03cde03786eb8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fe5159febd2c4b73a1d181c12051d1c6, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/c44d23b185564f918e691852be861797] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=49.0 K 2024-11-23T15:40:34,410 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7dce56890dd4763a7919e2816206887, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732376430293 2024-11-23T15:40:34,411 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c918ef77a6548548f38bb06aaf649b7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732376430293 2024-11-23T15:40:34,411 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b1c39a157794e36b0921adc9caff695, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732376430839 2024-11-23T15:40:34,411 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 529b33480c0f447198a03cde03786eb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732376430839 2024-11-23T15:40:34,412 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8091e03607e2442c95de26b7851df432, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1732376431467 2024-11-23T15:40:34,412 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26c0f4119ffe473d8b51b4aa52a9800b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1732376431872 2024-11-23T15:40:34,412 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting fe5159febd2c4b73a1d181c12051d1c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1732376431467 2024-11-23T15:40:34,413 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting c44d23b185564f918e691852be861797, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1732376431872 2024-11-23T15:40:34,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-23T15:40:34,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:34,423 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-23T15:40:34,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/7bd12afffe8c44c181f9bf085dfb9431 is 50, key is test_row_0/A:col10/1732376433047/Put/seqid=0 2024-11-23T15:40:34,457 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#B#compaction#103 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:34,458 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/ae037623d815476a868e03315c7c57b5 is 50, key is test_row_0/B:col10/1732376431888/Put/seqid=0 2024-11-23T15:40:34,458 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#A#compaction#104 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:34,458 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/aa41520ce90a430499144437af1654cf is 50, key is test_row_0/A:col10/1732376431888/Put/seqid=0 2024-11-23T15:40:34,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741939_1115 (size=12301) 2024-11-23T15:40:34,480 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/7bd12afffe8c44c181f9bf085dfb9431 2024-11-23T15:40:34,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741940_1116 (size=13459) 2024-11-23T15:40:34,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741941_1117 (size=13459) 2024-11-23T15:40:34,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/26a76191e26f45ae967d2dce6e6d34e6 is 50, key is test_row_0/B:col10/1732376433047/Put/seqid=0 2024-11-23T15:40:34,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741942_1118 (size=12301) 2024-11-23T15:40:34,896 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/aa41520ce90a430499144437af1654cf as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/aa41520ce90a430499144437af1654cf 2024-11-23T15:40:34,902 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/ae037623d815476a868e03315c7c57b5 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/ae037623d815476a868e03315c7c57b5 2024-11-23T15:40:34,903 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/26a76191e26f45ae967d2dce6e6d34e6 2024-11-23T15:40:34,904 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/A of dad1202300d1ddd3a95c32af1ad98ed0 into aa41520ce90a430499144437af1654cf(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:34,904 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:34,904 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/A, priority=12, startTime=1732376434405; duration=0sec 2024-11-23T15:40:34,904 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:34,904 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:A 2024-11-23T15:40:34,904 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:40:34,909 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:40:34,909 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/C is initiating minor compaction (all files) 2024-11-23T15:40:34,909 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/C in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:34,910 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/ddf1a5219af544499dbac5d8dd7f5e50, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/44d0a249fbea40ea97630ccf358763d0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/f470a05edc104e35ab787f6588170ab4, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/119d66470b4941749c6fe30326140f61] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=49.0 K 2024-11-23T15:40:34,911 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/B of dad1202300d1ddd3a95c32af1ad98ed0 into ae037623d815476a868e03315c7c57b5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:34,911 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:34,911 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/B, priority=12, startTime=1732376434405; duration=0sec 2024-11-23T15:40:34,911 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:34,911 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:B 2024-11-23T15:40:34,911 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddf1a5219af544499dbac5d8dd7f5e50, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732376430293 2024-11-23T15:40:34,912 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44d0a249fbea40ea97630ccf358763d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732376430839 2024-11-23T15:40:34,914 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting f470a05edc104e35ab787f6588170ab4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1732376431467 2024-11-23T15:40:34,915 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 119d66470b4941749c6fe30326140f61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1732376431872 2024-11-23T15:40:34,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/d42b1b7d1ac9489e97920353ffc43505 is 50, key is test_row_0/C:col10/1732376433047/Put/seqid=0 2024-11-23T15:40:34,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741943_1119 (size=12301) 2024-11-23T15:40:34,923 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/d42b1b7d1ac9489e97920353ffc43505 2024-11-23T15:40:34,927 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#C#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:34,927 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/4eb8cfa150194689b616472cc21eaab9 is 50, key is test_row_0/C:col10/1732376431888/Put/seqid=0 2024-11-23T15:40:34,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/7bd12afffe8c44c181f9bf085dfb9431 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/7bd12afffe8c44c181f9bf085dfb9431 2024-11-23T15:40:34,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741944_1120 (size=13459) 2024-11-23T15:40:34,936 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/7bd12afffe8c44c181f9bf085dfb9431, entries=150, sequenceid=481, filesize=12.0 K 2024-11-23T15:40:34,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/26a76191e26f45ae967d2dce6e6d34e6 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/26a76191e26f45ae967d2dce6e6d34e6 2024-11-23T15:40:34,943 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/26a76191e26f45ae967d2dce6e6d34e6, entries=150, sequenceid=481, filesize=12.0 K 2024-11-23T15:40:34,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/d42b1b7d1ac9489e97920353ffc43505 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/d42b1b7d1ac9489e97920353ffc43505 2024-11-23T15:40:34,954 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/d42b1b7d1ac9489e97920353ffc43505, entries=150, sequenceid=481, filesize=12.0 K 2024-11-23T15:40:34,956 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for dad1202300d1ddd3a95c32af1ad98ed0 in 533ms, sequenceid=481, compaction requested=false 2024-11-23T15:40:34,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:34,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:34,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-23T15:40:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-23T15:40:34,972 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-23T15:40:34,972 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7820 sec 2024-11-23T15:40:34,976 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.7880 sec 2024-11-23T15:40:35,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:35,117 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:40:35,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T15:40:35,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/6d4929d881ad45ecbf8c317c69ac11be is 50, key is test_row_0/A:col10/1732376435114/Put/seqid=0 2024-11-23T15:40:35,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741945_1121 (size=14741) 2024-11-23T15:40:35,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376495219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:35,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:35,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:35,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376495221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:35,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:35,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:35,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:35,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:35,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55398 deadline: 1732376495224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:35,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:35,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55392 deadline: 1732376495224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:35,225 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_1' in dad1202300d1ddd3a95c32af1ad98ed0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:40:35,225 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_1' in dad1202300d1ddd3a95c32af1ad98ed0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:40:35,225 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:35,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:35,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:35,226 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:35,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:35,226 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:35,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:35,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376495335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:35,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376495335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:35,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T15:40:35,345 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:40:35,346 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:40:35,348 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/4eb8cfa150194689b616472cc21eaab9 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/4eb8cfa150194689b616472cc21eaab9 2024-11-23T15:40:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees 2024-11-23T15:40:35,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-23T15:40:35,351 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:40:35,353 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:40:35,353 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:40:35,362 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/C of dad1202300d1ddd3a95c32af1ad98ed0 into 4eb8cfa150194689b616472cc21eaab9(size=13.1 K), total size for store is 25.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:35,362 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:35,362 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/C, priority=12, startTime=1732376434406; duration=0sec 2024-11-23T15:40:35,362 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:35,362 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:C 2024-11-23T15:40:35,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-23T15:40:35,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-23T15:40:35,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:35,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:35,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:35,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:35,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:35,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:35,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/6d4929d881ad45ecbf8c317c69ac11be 2024-11-23T15:40:35,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376495544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:35,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:35,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376495544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:35,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:35,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:35,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:35,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/8f7ecd23b4b3430bae2864b19bf40268 is 50, key is test_row_0/B:col10/1732376435114/Put/seqid=0 2024-11-23T15:40:35,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741946_1122 (size=12301) 2024-11-23T15:40:35,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-23T15:40:35,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:35,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:35,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:35,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:35,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:35,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-23T15:40:35,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-23T15:40:35,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:35,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:35,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:35,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:35,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:35,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376495854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:35,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:35,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376495864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:35,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:35,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:35,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:35,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-23T15:40:35,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:35,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:35,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:35,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:35,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-23T15:40:36,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/8f7ecd23b4b3430bae2864b19bf40268 2024-11-23T15:40:36,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/56a4418283584a9dacd0a8898225edbf is 50, key is test_row_0/C:col10/1732376435114/Put/seqid=0 2024-11-23T15:40:36,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741947_1123 (size=12301) 2024-11-23T15:40:36,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-23T15:40:36,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:36,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:36,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:36,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:36,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:36,186 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-23T15:40:36,186 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:40:36,186 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-23T15:40:36,186 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:40:36,186 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-23T15:40:36,186 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:40:36,188 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-23T15:40:36,188 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:40:36,191 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:36,191 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:36,191 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:36,191 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:36,191 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:36,191 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:36,191 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:36,191 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:40:36,191 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:40:36,191 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:40:36,192 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:36,192 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:40:36,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-23T15:40:36,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:36,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:36,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:36,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:36,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:36,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55414 deadline: 1732376496364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:36,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55430 deadline: 1732376496375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:36,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:36,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:36,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:40:36,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/56a4418283584a9dacd0a8898225edbf 2024-11-23T15:40:36,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/6d4929d881ad45ecbf8c317c69ac11be as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6d4929d881ad45ecbf8c317c69ac11be 2024-11-23T15:40:36,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-23T15:40:36,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:36,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:36,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:36,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] handler.RSProcedureHandler(58): pid=30 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:36,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=30 java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:36,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6d4929d881ad45ecbf8c317c69ac11be, entries=200, sequenceid=494, filesize=14.4 K 2024-11-23T15:40:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=30 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:36,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/8f7ecd23b4b3430bae2864b19bf40268 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/8f7ecd23b4b3430bae2864b19bf40268 2024-11-23T15:40:36,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/8f7ecd23b4b3430bae2864b19bf40268, entries=150, sequenceid=494, filesize=12.0 K 2024-11-23T15:40:36,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/56a4418283584a9dacd0a8898225edbf as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/56a4418283584a9dacd0a8898225edbf 2024-11-23T15:40:36,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/56a4418283584a9dacd0a8898225edbf, entries=150, sequenceid=494, filesize=12.0 K 2024-11-23T15:40:36,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for dad1202300d1ddd3a95c32af1ad98ed0 in 1321ms, sequenceid=494, compaction requested=true 2024-11-23T15:40:36,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:36,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:40:36,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:36,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:40:36,438 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:36,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:36,438 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:36,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dad1202300d1ddd3a95c32af1ad98ed0:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:40:36,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:36,439 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:36,439 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:36,439 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/B is initiating minor compaction (all files) 2024-11-23T15:40:36,439 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/A is initiating minor compaction (all files) 2024-11-23T15:40:36,439 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/B in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:36,439 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/A in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:36,439 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/ae037623d815476a868e03315c7c57b5, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/26a76191e26f45ae967d2dce6e6d34e6, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/8f7ecd23b4b3430bae2864b19bf40268] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=37.2 K 2024-11-23T15:40:36,439 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/aa41520ce90a430499144437af1654cf, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/7bd12afffe8c44c181f9bf085dfb9431, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6d4929d881ad45ecbf8c317c69ac11be] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=39.6 K 2024-11-23T15:40:36,439 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae037623d815476a868e03315c7c57b5, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1732376431872 2024-11-23T15:40:36,439 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting aa41520ce90a430499144437af1654cf, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1732376431872 2024-11-23T15:40:36,440 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26a76191e26f45ae967d2dce6e6d34e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732376433047 2024-11-23T15:40:36,440 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bd12afffe8c44c181f9bf085dfb9431, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732376433047 2024-11-23T15:40:36,440 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f7ecd23b4b3430bae2864b19bf40268, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1732376435103 2024-11-23T15:40:36,440 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d4929d881ad45ecbf8c317c69ac11be, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1732376435099 2024-11-23T15:40:36,447 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#B#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:36,448 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#A#compaction#112 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:36,448 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/99dab134fd24493da8fa70b13b756e0d is 50, key is test_row_0/A:col10/1732376435114/Put/seqid=0 2024-11-23T15:40:36,448 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/509b318d367c4e59b5529cf579b3563a is 50, key is test_row_0/B:col10/1732376435114/Put/seqid=0 2024-11-23T15:40:36,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741949_1125 (size=13561) 2024-11-23T15:40:36,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741948_1124 (size=13561) 2024-11-23T15:40:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-23T15:40:36,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-23T15:40:36,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:36,579 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T15:40:36,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/7eda309982e6435c8a5ea41b877d1dc1 is 50, key is test_row_0/A:col10/1732376435219/Put/seqid=0 2024-11-23T15:40:36,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741950_1126 (size=12301) 2024-11-23T15:40:36,858 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/509b318d367c4e59b5529cf579b3563a as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/509b318d367c4e59b5529cf579b3563a 2024-11-23T15:40:36,865 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/99dab134fd24493da8fa70b13b756e0d as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/99dab134fd24493da8fa70b13b756e0d 2024-11-23T15:40:36,870 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/B of dad1202300d1ddd3a95c32af1ad98ed0 into 509b318d367c4e59b5529cf579b3563a(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:36,870 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:36,870 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/B, priority=13, startTime=1732376436438; duration=0sec 2024-11-23T15:40:36,870 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:36,870 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:B 2024-11-23T15:40:36,871 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:36,872 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:36,872 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): dad1202300d1ddd3a95c32af1ad98ed0/C is initiating minor compaction (all files) 2024-11-23T15:40:36,872 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/A of dad1202300d1ddd3a95c32af1ad98ed0 into 99dab134fd24493da8fa70b13b756e0d(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:36,872 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:36,873 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/A, priority=13, startTime=1732376436438; duration=0sec 2024-11-23T15:40:36,873 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dad1202300d1ddd3a95c32af1ad98ed0/C in TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:36,873 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:36,873 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:A 2024-11-23T15:40:36,873 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/4eb8cfa150194689b616472cc21eaab9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/d42b1b7d1ac9489e97920353ffc43505, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/56a4418283584a9dacd0a8898225edbf] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp, totalSize=37.2 K 2024-11-23T15:40:36,873 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4eb8cfa150194689b616472cc21eaab9, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1732376431872 2024-11-23T15:40:36,874 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting d42b1b7d1ac9489e97920353ffc43505, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732376433047 2024-11-23T15:40:36,874 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56a4418283584a9dacd0a8898225edbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1732376435103 2024-11-23T15:40:36,884 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dad1202300d1ddd3a95c32af1ad98ed0#C#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:36,885 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/771e94afc80e4ecab0f67bdde986f5f7 is 50, key is test_row_0/C:col10/1732376435114/Put/seqid=0 2024-11-23T15:40:36,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741951_1127 (size=13561) 2024-11-23T15:40:36,902 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/771e94afc80e4ecab0f67bdde986f5f7 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/771e94afc80e4ecab0f67bdde986f5f7 2024-11-23T15:40:36,908 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dad1202300d1ddd3a95c32af1ad98ed0/C of dad1202300d1ddd3a95c32af1ad98ed0 into 771e94afc80e4ecab0f67bdde986f5f7(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:36,908 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:36,908 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., storeName=dad1202300d1ddd3a95c32af1ad98ed0/C, priority=13, startTime=1732376436438; duration=0sec 2024-11-23T15:40:36,908 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:36,908 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dad1202300d1ddd3a95c32af1ad98ed0:C 2024-11-23T15:40:36,996 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/7eda309982e6435c8a5ea41b877d1dc1 2024-11-23T15:40:37,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/1b7ecfc625d841c487ce4f996d6b9944 is 50, key is test_row_0/B:col10/1732376435219/Put/seqid=0 2024-11-23T15:40:37,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741952_1128 (size=12301) 2024-11-23T15:40:37,013 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/1b7ecfc625d841c487ce4f996d6b9944 2024-11-23T15:40:37,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/79cd25aa302342a996b071bfa30d7716 is 50, key is test_row_0/C:col10/1732376435219/Put/seqid=0 2024-11-23T15:40:37,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741953_1129 (size=12301) 2024-11-23T15:40:37,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:37,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. as already flushing 2024-11-23T15:40:37,236 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-23T15:40:37,236 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-23T15:40:37,236 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:40:37,236 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:40:37,236 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:37,236 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:37,236 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:37,236 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:37,236 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:40:37,236 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:40:37,376 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-23T15:40:37,376 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:40:37,376 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:37,376 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:37,376 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:40:37,386 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-23T15:40:37,386 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:40:37,386 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:37,386 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:37,386 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:40:37,424 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/79cd25aa302342a996b071bfa30d7716 2024-11-23T15:40:37,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/7eda309982e6435c8a5ea41b877d1dc1 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/7eda309982e6435c8a5ea41b877d1dc1 2024-11-23T15:40:37,435 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/7eda309982e6435c8a5ea41b877d1dc1, entries=150, sequenceid=520, filesize=12.0 K 2024-11-23T15:40:37,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/1b7ecfc625d841c487ce4f996d6b9944 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1b7ecfc625d841c487ce4f996d6b9944 2024-11-23T15:40:37,442 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1b7ecfc625d841c487ce4f996d6b9944, entries=150, sequenceid=520, filesize=12.0 K 2024-11-23T15:40:37,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/79cd25aa302342a996b071bfa30d7716 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/79cd25aa302342a996b071bfa30d7716 2024-11-23T15:40:37,449 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/79cd25aa302342a996b071bfa30d7716, entries=150, sequenceid=520, filesize=12.0 K 2024-11-23T15:40:37,450 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=26.84 KB/27480 for dad1202300d1ddd3a95c32af1ad98ed0 in 870ms, sequenceid=520, compaction requested=false 2024-11-23T15:40:37,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for dad1202300d1ddd3a95c32af1ad98ed0: 2024-11-23T15:40:37,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:37,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-11-23T15:40:37,450 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T15:40:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-11-23T15:40:37,453 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-23T15:40:37,453 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0980 sec 2024-11-23T15:40:37,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees in 2.1080 sec 2024-11-23T15:40:37,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-23T15:40:37,495 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:40:42,569 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-23T15:40:42,570 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:40:42,570 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:42,570 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:42,570 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:40:42,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T15:40:42,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 120 2024-11-23T15:40:42,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-23T15:40:42,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-23T15:40:42,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-23T15:40:42,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-23T15:40:42,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T15:40:42,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3018 2024-11-23T15:40:42,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2921 2024-11-23T15:40:42,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T15:40:42,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1429 2024-11-23T15:40:42,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4281 rows 2024-11-23T15:40:42,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1441 2024-11-23T15:40:42,571 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4322 rows 2024-11-23T15:40:42,572 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T15:40:42,572 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T15:40:42,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:42,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:42,572 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:40:42,576 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-23T15:40:42,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T15:40:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-23T15:40:42,583 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376442583"}]},"ts":"1732376442583"} 2024-11-23T15:40:42,585 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T15:40:42,585 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T15:40:42,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T15:40:42,591 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dad1202300d1ddd3a95c32af1ad98ed0, UNASSIGN}] 2024-11-23T15:40:42,592 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dad1202300d1ddd3a95c32af1ad98ed0, UNASSIGN 2024-11-23T15:40:42,593 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=dad1202300d1ddd3a95c32af1ad98ed0, regionState=CLOSING, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:42,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dad1202300d1ddd3a95c32af1ad98ed0, UNASSIGN because future has completed 2024-11-23T15:40:42,596 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T15:40:42,596 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194}] 2024-11-23T15:40:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-23T15:40:42,754 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:42,754 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T15:40:42,755 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing dad1202300d1ddd3a95c32af1ad98ed0, disabling compactions & flushes 2024-11-23T15:40:42,755 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:42,755 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:42,755 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. after waiting 0 ms 2024-11-23T15:40:42,755 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:42,755 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(2902): Flushing dad1202300d1ddd3a95c32af1ad98ed0 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-23T15:40:42,761 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/21749f1299f54ee38d643e2ebbff8148 is 50, key is test_row_1/A:col10/1732376442565/Put/seqid=0 2024-11-23T15:40:42,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741954_1130 (size=9857) 2024-11-23T15:40:42,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-23T15:40:43,180 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/21749f1299f54ee38d643e2ebbff8148 2024-11-23T15:40:43,192 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/5f94f978e0204dde9ec574ea6e3a94b1 is 50, key is test_row_1/B:col10/1732376442565/Put/seqid=0 2024-11-23T15:40:43,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741955_1131 (size=9857) 2024-11-23T15:40:43,206 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/5f94f978e0204dde9ec574ea6e3a94b1 2024-11-23T15:40:43,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-23T15:40:43,216 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/d7c174d0991a4eceacce63bdd25b2e75 is 50, key is test_row_1/C:col10/1732376442565/Put/seqid=0 2024-11-23T15:40:43,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741956_1132 (size=9857) 2024-11-23T15:40:43,629 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/d7c174d0991a4eceacce63bdd25b2e75 2024-11-23T15:40:43,637 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/A/21749f1299f54ee38d643e2ebbff8148 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/21749f1299f54ee38d643e2ebbff8148 2024-11-23T15:40:43,643 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/21749f1299f54ee38d643e2ebbff8148, entries=100, sequenceid=531, filesize=9.6 K 2024-11-23T15:40:43,644 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/B/5f94f978e0204dde9ec574ea6e3a94b1 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/5f94f978e0204dde9ec574ea6e3a94b1 2024-11-23T15:40:43,651 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/5f94f978e0204dde9ec574ea6e3a94b1, entries=100, sequenceid=531, filesize=9.6 K 2024-11-23T15:40:43,652 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/.tmp/C/d7c174d0991a4eceacce63bdd25b2e75 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/d7c174d0991a4eceacce63bdd25b2e75 2024-11-23T15:40:43,658 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/d7c174d0991a4eceacce63bdd25b2e75, entries=100, sequenceid=531, filesize=9.6 K 2024-11-23T15:40:43,659 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for dad1202300d1ddd3a95c32af1ad98ed0 in 903ms, sequenceid=531, compaction requested=true 2024-11-23T15:40:43,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/73a6a0e96b414aaabef28f7c85c56f7c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/63e61ab664754f788794927cd8ba9664, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/efc6113e26a64e20b9fd047d06966d60, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/5ce8afcb0fcd4c1b963bc638aa8ccb15, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/c2ac6e16d6fe4cfc90a4660b6998f9a5, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/a8818f386cfd40cc8392621c57571876, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6069b726b76946d89881845a19a77f71, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/0b3291f0ab5045a3b3ed725fcd5a92f2, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/583060b9da95457a8f75305020fc3691, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/7b430c6731414f9fae66bc4cd9949432, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6bf7154f813347ee9c5428783b1fc441, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/bf69bc0559e14ae6b31d974493578ad2, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/7324e307a8d74712ad2ac36111309262, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/1e6717fb19044bd0822a44639c0955fe, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/76b098206e8e417fb1b119a832ca53b7, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/e0059d94e89b4b3fbb6ea1ec56b3146a, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/a04b046fcb914ac78ecf36032353fe23, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/93516305105f4f3fa01f7cc73b15c787, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/946f4e52b9da453492915094f8030925, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/44ee99398d0447848ccdd25e824782fa, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/039cdb3fdb174f9896c4b2ccf8b6762e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/74b5df2b752d42e4b72345b34139cfc8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/edfc944151564e11ac24bb48193baace, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/de6201e4adc740ffa7962fd42b0e57c8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/0c4597e47bbd4918b328f88a7543cede, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/0013c0b76e824aaf9df9ed9f210b18ee, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/be645cafb7044cd6885af4508d3e7752, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/ff5f981f6db241cd95beb8f4a7c47790, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/cb2f94148dae422c8d5934c9986e464c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/a060c98acbc34281af60303ddb9b047c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/b7dce56890dd4763a7919e2816206887, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6b1c39a157794e36b0921adc9caff695, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/8091e03607e2442c95de26b7851df432, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/aa41520ce90a430499144437af1654cf, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/26c0f4119ffe473d8b51b4aa52a9800b, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/7bd12afffe8c44c181f9bf085dfb9431, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6d4929d881ad45ecbf8c317c69ac11be] to archive 2024-11-23T15:40:43,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T15:40:43,669 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/73a6a0e96b414aaabef28f7c85c56f7c to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/73a6a0e96b414aaabef28f7c85c56f7c 2024-11-23T15:40:43,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/63e61ab664754f788794927cd8ba9664 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/63e61ab664754f788794927cd8ba9664 2024-11-23T15:40:43,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/efc6113e26a64e20b9fd047d06966d60 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/efc6113e26a64e20b9fd047d06966d60 2024-11-23T15:40:43,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/5ce8afcb0fcd4c1b963bc638aa8ccb15 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/5ce8afcb0fcd4c1b963bc638aa8ccb15 2024-11-23T15:40:43,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/c2ac6e16d6fe4cfc90a4660b6998f9a5 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/c2ac6e16d6fe4cfc90a4660b6998f9a5 2024-11-23T15:40:43,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/a8818f386cfd40cc8392621c57571876 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/a8818f386cfd40cc8392621c57571876 2024-11-23T15:40:43,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6069b726b76946d89881845a19a77f71 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6069b726b76946d89881845a19a77f71 2024-11-23T15:40:43,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/0b3291f0ab5045a3b3ed725fcd5a92f2 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/0b3291f0ab5045a3b3ed725fcd5a92f2 2024-11-23T15:40:43,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/583060b9da95457a8f75305020fc3691 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/583060b9da95457a8f75305020fc3691 2024-11-23T15:40:43,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/7b430c6731414f9fae66bc4cd9949432 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/7b430c6731414f9fae66bc4cd9949432 2024-11-23T15:40:43,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6bf7154f813347ee9c5428783b1fc441 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6bf7154f813347ee9c5428783b1fc441 2024-11-23T15:40:43,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/bf69bc0559e14ae6b31d974493578ad2 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/bf69bc0559e14ae6b31d974493578ad2 2024-11-23T15:40:43,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/7324e307a8d74712ad2ac36111309262 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/7324e307a8d74712ad2ac36111309262 2024-11-23T15:40:43,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/1e6717fb19044bd0822a44639c0955fe to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/1e6717fb19044bd0822a44639c0955fe 2024-11-23T15:40:43,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/76b098206e8e417fb1b119a832ca53b7 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/76b098206e8e417fb1b119a832ca53b7 2024-11-23T15:40:43,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/e0059d94e89b4b3fbb6ea1ec56b3146a to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/e0059d94e89b4b3fbb6ea1ec56b3146a 2024-11-23T15:40:43,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/a04b046fcb914ac78ecf36032353fe23 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/a04b046fcb914ac78ecf36032353fe23 2024-11-23T15:40:43,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/93516305105f4f3fa01f7cc73b15c787 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/93516305105f4f3fa01f7cc73b15c787 2024-11-23T15:40:43,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/946f4e52b9da453492915094f8030925 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/946f4e52b9da453492915094f8030925 2024-11-23T15:40:43,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/44ee99398d0447848ccdd25e824782fa to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/44ee99398d0447848ccdd25e824782fa 2024-11-23T15:40:43,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/039cdb3fdb174f9896c4b2ccf8b6762e to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/039cdb3fdb174f9896c4b2ccf8b6762e 2024-11-23T15:40:43,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/74b5df2b752d42e4b72345b34139cfc8 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/74b5df2b752d42e4b72345b34139cfc8 2024-11-23T15:40:43,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/edfc944151564e11ac24bb48193baace to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/edfc944151564e11ac24bb48193baace 2024-11-23T15:40:43,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/de6201e4adc740ffa7962fd42b0e57c8 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/de6201e4adc740ffa7962fd42b0e57c8 2024-11-23T15:40:43,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/0c4597e47bbd4918b328f88a7543cede to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/0c4597e47bbd4918b328f88a7543cede 2024-11-23T15:40:43,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/0013c0b76e824aaf9df9ed9f210b18ee to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/0013c0b76e824aaf9df9ed9f210b18ee 2024-11-23T15:40:43,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/be645cafb7044cd6885af4508d3e7752 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/be645cafb7044cd6885af4508d3e7752 2024-11-23T15:40:43,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/ff5f981f6db241cd95beb8f4a7c47790 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/ff5f981f6db241cd95beb8f4a7c47790 2024-11-23T15:40:43,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/cb2f94148dae422c8d5934c9986e464c to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/cb2f94148dae422c8d5934c9986e464c 2024-11-23T15:40:43,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-23T15:40:43,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/a060c98acbc34281af60303ddb9b047c to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/a060c98acbc34281af60303ddb9b047c 2024-11-23T15:40:43,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/b7dce56890dd4763a7919e2816206887 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/b7dce56890dd4763a7919e2816206887 2024-11-23T15:40:43,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6b1c39a157794e36b0921adc9caff695 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6b1c39a157794e36b0921adc9caff695 2024-11-23T15:40:43,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/8091e03607e2442c95de26b7851df432 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/8091e03607e2442c95de26b7851df432 2024-11-23T15:40:43,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/aa41520ce90a430499144437af1654cf to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/aa41520ce90a430499144437af1654cf 2024-11-23T15:40:43,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/26c0f4119ffe473d8b51b4aa52a9800b to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/26c0f4119ffe473d8b51b4aa52a9800b 2024-11-23T15:40:43,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/7bd12afffe8c44c181f9bf085dfb9431 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/7bd12afffe8c44c181f9bf085dfb9431 2024-11-23T15:40:43,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6d4929d881ad45ecbf8c317c69ac11be to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/6d4929d881ad45ecbf8c317c69ac11be 2024-11-23T15:40:43,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1b81c6191be54549b95dd10555d3ff90, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/b50660d2fb76453bbed56342e730f28d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/9544fc4582274bb6aecf607718f54c1b, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3c7c59471d014e0b84cb00e8f6e0cf89, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/8df211a484e04a318fbf5ceff704e9db, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/eacec568583e4c3582a5598b21a7623f, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/447159a9bd3346ed861775dd6856c8fb, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fb1b1e13127d4e42b485d7d7ede1ac98, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/0b0f88c84c1c43c2a838280475e7a174, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/7501669030644e41808ba47ca82680a0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3580e0c343094aff9ef27438660b356c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/81e8569b11b34244bc3df0fd238d0c6f, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3f61fb5c7b3b4f92a29386d8166f1b7f, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/cdbf86d562a0472d8362d517e72ee77b, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/f9ef2a573bf3462e9ba1b414a5ad6f5a, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/d336e825e46a45b7b134496887cdcfeb, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/287df1bce8d947699350d176354971dd, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3b7d1cf802744cd09518b81758d506e0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1b2f75fb3eb94f728a116fac245908e0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fd5e69ec002a4e73b1620dd52a87e2df, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1a32ac83432b4f449a7c38e6b24f3aae, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/130a75da0a844b00886697ee514269f8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3ad8f7e42cc544428c007c824416e325, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/6b0f22d43fe347c98ecf74623d108bdc, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/984b38124ed745caa0f7903e8371efac, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/cebf219fbd21477b85624fc0c0641e58, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1cfa865176de4f8e95ed7a61e215b9a4, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/5afafbe8689143338b3e699f4bc3b0d2, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/c7bcb4729d9f4839b27e0a21fc4e36d9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3c918ef77a6548548f38bb06aaf649b7, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fa3f43165b43449fa4a63a00e288b749, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/529b33480c0f447198a03cde03786eb8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fe5159febd2c4b73a1d181c12051d1c6, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/ae037623d815476a868e03315c7c57b5, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/c44d23b185564f918e691852be861797, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/26a76191e26f45ae967d2dce6e6d34e6, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/8f7ecd23b4b3430bae2864b19bf40268] to archive 2024-11-23T15:40:43,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T15:40:43,768 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1b81c6191be54549b95dd10555d3ff90 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1b81c6191be54549b95dd10555d3ff90 2024-11-23T15:40:43,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/b50660d2fb76453bbed56342e730f28d to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/b50660d2fb76453bbed56342e730f28d 2024-11-23T15:40:43,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/9544fc4582274bb6aecf607718f54c1b to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/9544fc4582274bb6aecf607718f54c1b 2024-11-23T15:40:43,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3c7c59471d014e0b84cb00e8f6e0cf89 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3c7c59471d014e0b84cb00e8f6e0cf89 2024-11-23T15:40:43,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/8df211a484e04a318fbf5ceff704e9db to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/8df211a484e04a318fbf5ceff704e9db 2024-11-23T15:40:43,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/eacec568583e4c3582a5598b21a7623f to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/eacec568583e4c3582a5598b21a7623f 2024-11-23T15:40:43,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/447159a9bd3346ed861775dd6856c8fb to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/447159a9bd3346ed861775dd6856c8fb 2024-11-23T15:40:43,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fb1b1e13127d4e42b485d7d7ede1ac98 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fb1b1e13127d4e42b485d7d7ede1ac98 2024-11-23T15:40:43,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/0b0f88c84c1c43c2a838280475e7a174 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/0b0f88c84c1c43c2a838280475e7a174 2024-11-23T15:40:43,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/7501669030644e41808ba47ca82680a0 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/7501669030644e41808ba47ca82680a0 2024-11-23T15:40:43,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3580e0c343094aff9ef27438660b356c to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3580e0c343094aff9ef27438660b356c 2024-11-23T15:40:43,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/81e8569b11b34244bc3df0fd238d0c6f to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/81e8569b11b34244bc3df0fd238d0c6f 2024-11-23T15:40:43,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3f61fb5c7b3b4f92a29386d8166f1b7f to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3f61fb5c7b3b4f92a29386d8166f1b7f 2024-11-23T15:40:43,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/cdbf86d562a0472d8362d517e72ee77b to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/cdbf86d562a0472d8362d517e72ee77b 2024-11-23T15:40:43,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/f9ef2a573bf3462e9ba1b414a5ad6f5a to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/f9ef2a573bf3462e9ba1b414a5ad6f5a 2024-11-23T15:40:43,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/d336e825e46a45b7b134496887cdcfeb to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/d336e825e46a45b7b134496887cdcfeb 2024-11-23T15:40:43,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/287df1bce8d947699350d176354971dd to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/287df1bce8d947699350d176354971dd 2024-11-23T15:40:43,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3b7d1cf802744cd09518b81758d506e0 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3b7d1cf802744cd09518b81758d506e0 2024-11-23T15:40:43,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1b2f75fb3eb94f728a116fac245908e0 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1b2f75fb3eb94f728a116fac245908e0 2024-11-23T15:40:43,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fd5e69ec002a4e73b1620dd52a87e2df to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fd5e69ec002a4e73b1620dd52a87e2df 2024-11-23T15:40:43,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1a32ac83432b4f449a7c38e6b24f3aae to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1a32ac83432b4f449a7c38e6b24f3aae 2024-11-23T15:40:43,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/130a75da0a844b00886697ee514269f8 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/130a75da0a844b00886697ee514269f8 2024-11-23T15:40:43,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3ad8f7e42cc544428c007c824416e325 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3ad8f7e42cc544428c007c824416e325 2024-11-23T15:40:43,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/6b0f22d43fe347c98ecf74623d108bdc to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/6b0f22d43fe347c98ecf74623d108bdc 2024-11-23T15:40:43,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/984b38124ed745caa0f7903e8371efac to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/984b38124ed745caa0f7903e8371efac 2024-11-23T15:40:43,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/cebf219fbd21477b85624fc0c0641e58 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/cebf219fbd21477b85624fc0c0641e58 2024-11-23T15:40:43,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1cfa865176de4f8e95ed7a61e215b9a4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1cfa865176de4f8e95ed7a61e215b9a4 2024-11-23T15:40:43,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/5afafbe8689143338b3e699f4bc3b0d2 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/5afafbe8689143338b3e699f4bc3b0d2 2024-11-23T15:40:43,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/c7bcb4729d9f4839b27e0a21fc4e36d9 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/c7bcb4729d9f4839b27e0a21fc4e36d9 2024-11-23T15:40:43,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3c918ef77a6548548f38bb06aaf649b7 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/3c918ef77a6548548f38bb06aaf649b7 2024-11-23T15:40:43,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fa3f43165b43449fa4a63a00e288b749 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fa3f43165b43449fa4a63a00e288b749 2024-11-23T15:40:43,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/529b33480c0f447198a03cde03786eb8 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/529b33480c0f447198a03cde03786eb8 2024-11-23T15:40:43,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fe5159febd2c4b73a1d181c12051d1c6 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/fe5159febd2c4b73a1d181c12051d1c6 2024-11-23T15:40:43,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/ae037623d815476a868e03315c7c57b5 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/ae037623d815476a868e03315c7c57b5 2024-11-23T15:40:43,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/c44d23b185564f918e691852be861797 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/c44d23b185564f918e691852be861797 2024-11-23T15:40:43,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/26a76191e26f45ae967d2dce6e6d34e6 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/26a76191e26f45ae967d2dce6e6d34e6 2024-11-23T15:40:43,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/8f7ecd23b4b3430bae2864b19bf40268 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/8f7ecd23b4b3430bae2864b19bf40268 2024-11-23T15:40:43,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/de97edf6edbe4e23ac8d427cebdfd0a7, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/4cea9178b818483ea36ba92eddfeb936, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/af0203bf575b455fae81cc7024f60e16, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/c64b38f8ebe647aeb3d2659b1f1bf10e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/48dfa0c4c0a847428caa7d3464455598, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/afe37d6bae39494dad7b86cb834b5bee, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/95b835c6375a4e77988cceb7db2edfef, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/b1ee220b8bdd4c2bab2145ca65c425f1, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/8e3ab70751d542a5be8db0e022cb3e88, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/fef07cada93743258969cad3113f0f70, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/3a86b3fb212741bd8e6f7a3f621e1f86, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/2bb8ca1a852c48b3ba2bc19f8f299d68, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/b2c59c35ff99486b926603779d00b47a, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/d432d2037fdb4454acf020761bcd66b6, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/7a120e8159f04093808ab3b75e36338b, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/941bb1dd86bb47fbb6a6f5b1764162e5, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/a32f23da560f486e9e000260108fdd9d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/a5dceffff1f0431f8432459c01d7e7a8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/64bc2bb373ba42d085e16860e401c63d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/15a8f9a2c23249cb8e73eeb2f6e27006, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/0d7f8c6a2c5d48889c45794555f74258, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/517dd48ad1ff4f96928e455d423c125e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/0e70c6a25b3b48b28e33a79ac7b6a025, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/df85719833b94f7b93916fb1e64924c0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/a85d0160095e49509f93acbd52fbaaf0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/58a726bac7b9457c9665c3e01bf83b23, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/02e8220a8f2c471884977e0132084388, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/61ae252469fe45ed970d7a60b8dd832c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/d5b902ef856c4b238403244f0cfab504, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/ddf1a5219af544499dbac5d8dd7f5e50, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/0c8336167d744cabb237a4dac8b2a376, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/44d0a249fbea40ea97630ccf358763d0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/f470a05edc104e35ab787f6588170ab4, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/4eb8cfa150194689b616472cc21eaab9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/119d66470b4941749c6fe30326140f61, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/d42b1b7d1ac9489e97920353ffc43505, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/56a4418283584a9dacd0a8898225edbf] to archive 2024-11-23T15:40:43,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T15:40:43,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/de97edf6edbe4e23ac8d427cebdfd0a7 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/de97edf6edbe4e23ac8d427cebdfd0a7 2024-11-23T15:40:43,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/4cea9178b818483ea36ba92eddfeb936 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/4cea9178b818483ea36ba92eddfeb936 2024-11-23T15:40:43,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/af0203bf575b455fae81cc7024f60e16 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/af0203bf575b455fae81cc7024f60e16 2024-11-23T15:40:43,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/c64b38f8ebe647aeb3d2659b1f1bf10e to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/c64b38f8ebe647aeb3d2659b1f1bf10e 2024-11-23T15:40:43,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/48dfa0c4c0a847428caa7d3464455598 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/48dfa0c4c0a847428caa7d3464455598 2024-11-23T15:40:43,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/afe37d6bae39494dad7b86cb834b5bee to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/afe37d6bae39494dad7b86cb834b5bee 2024-11-23T15:40:43,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/95b835c6375a4e77988cceb7db2edfef to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/95b835c6375a4e77988cceb7db2edfef 2024-11-23T15:40:43,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/b1ee220b8bdd4c2bab2145ca65c425f1 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/b1ee220b8bdd4c2bab2145ca65c425f1 2024-11-23T15:40:43,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/8e3ab70751d542a5be8db0e022cb3e88 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/8e3ab70751d542a5be8db0e022cb3e88 2024-11-23T15:40:43,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/fef07cada93743258969cad3113f0f70 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/fef07cada93743258969cad3113f0f70 2024-11-23T15:40:43,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/3a86b3fb212741bd8e6f7a3f621e1f86 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/3a86b3fb212741bd8e6f7a3f621e1f86 2024-11-23T15:40:43,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/2bb8ca1a852c48b3ba2bc19f8f299d68 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/2bb8ca1a852c48b3ba2bc19f8f299d68 2024-11-23T15:40:43,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/b2c59c35ff99486b926603779d00b47a to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/b2c59c35ff99486b926603779d00b47a 2024-11-23T15:40:43,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/d432d2037fdb4454acf020761bcd66b6 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/d432d2037fdb4454acf020761bcd66b6 2024-11-23T15:40:43,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/7a120e8159f04093808ab3b75e36338b to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/7a120e8159f04093808ab3b75e36338b 2024-11-23T15:40:43,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/941bb1dd86bb47fbb6a6f5b1764162e5 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/941bb1dd86bb47fbb6a6f5b1764162e5 2024-11-23T15:40:43,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/a32f23da560f486e9e000260108fdd9d to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/a32f23da560f486e9e000260108fdd9d 2024-11-23T15:40:43,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/a5dceffff1f0431f8432459c01d7e7a8 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/a5dceffff1f0431f8432459c01d7e7a8 2024-11-23T15:40:43,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/64bc2bb373ba42d085e16860e401c63d to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/64bc2bb373ba42d085e16860e401c63d 2024-11-23T15:40:43,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/15a8f9a2c23249cb8e73eeb2f6e27006 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/15a8f9a2c23249cb8e73eeb2f6e27006 2024-11-23T15:40:43,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/0d7f8c6a2c5d48889c45794555f74258 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/0d7f8c6a2c5d48889c45794555f74258 2024-11-23T15:40:43,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/517dd48ad1ff4f96928e455d423c125e to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/517dd48ad1ff4f96928e455d423c125e 2024-11-23T15:40:43,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/0e70c6a25b3b48b28e33a79ac7b6a025 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/0e70c6a25b3b48b28e33a79ac7b6a025 2024-11-23T15:40:43,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/df85719833b94f7b93916fb1e64924c0 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/df85719833b94f7b93916fb1e64924c0 2024-11-23T15:40:43,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/a85d0160095e49509f93acbd52fbaaf0 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/a85d0160095e49509f93acbd52fbaaf0 2024-11-23T15:40:43,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/58a726bac7b9457c9665c3e01bf83b23 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/58a726bac7b9457c9665c3e01bf83b23 2024-11-23T15:40:43,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/02e8220a8f2c471884977e0132084388 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/02e8220a8f2c471884977e0132084388 2024-11-23T15:40:43,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/61ae252469fe45ed970d7a60b8dd832c to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/61ae252469fe45ed970d7a60b8dd832c 2024-11-23T15:40:43,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/d5b902ef856c4b238403244f0cfab504 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/d5b902ef856c4b238403244f0cfab504 2024-11-23T15:40:43,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/ddf1a5219af544499dbac5d8dd7f5e50 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/ddf1a5219af544499dbac5d8dd7f5e50 2024-11-23T15:40:43,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/0c8336167d744cabb237a4dac8b2a376 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/0c8336167d744cabb237a4dac8b2a376 2024-11-23T15:40:43,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/44d0a249fbea40ea97630ccf358763d0 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/44d0a249fbea40ea97630ccf358763d0 2024-11-23T15:40:43,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/f470a05edc104e35ab787f6588170ab4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/f470a05edc104e35ab787f6588170ab4 2024-11-23T15:40:43,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/4eb8cfa150194689b616472cc21eaab9 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/4eb8cfa150194689b616472cc21eaab9 2024-11-23T15:40:43,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/119d66470b4941749c6fe30326140f61 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/119d66470b4941749c6fe30326140f61 2024-11-23T15:40:43,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/d42b1b7d1ac9489e97920353ffc43505 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/d42b1b7d1ac9489e97920353ffc43505 2024-11-23T15:40:43,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/56a4418283584a9dacd0a8898225edbf to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/56a4418283584a9dacd0a8898225edbf 2024-11-23T15:40:43,921 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/recovered.edits/534.seqid, newMaxSeqId=534, maxSeqId=1 2024-11-23T15:40:43,923 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. 2024-11-23T15:40:43,923 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for dad1202300d1ddd3a95c32af1ad98ed0: Waiting for close lock at 1732376442754Running coprocessor pre-close hooks at 1732376442754Disabling compacts and flushes for region at 1732376442754Disabling writes for close at 1732376442755 (+1 ms)Obtaining lock to block concurrent updates at 1732376442755Preparing flush snapshotting stores in dad1202300d1ddd3a95c32af1ad98ed0 at 1732376442755Finished memstore snapshotting TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732376442755Flushing stores of TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0. at 1732376442756 (+1 ms)Flushing dad1202300d1ddd3a95c32af1ad98ed0/A: creating writer at 1732376442756Flushing dad1202300d1ddd3a95c32af1ad98ed0/A: appending metadata at 1732376442761 (+5 ms)Flushing dad1202300d1ddd3a95c32af1ad98ed0/A: closing flushed file at 1732376442761Flushing dad1202300d1ddd3a95c32af1ad98ed0/B: creating writer at 1732376443187 (+426 ms)Flushing dad1202300d1ddd3a95c32af1ad98ed0/B: appending metadata at 1732376443191 (+4 ms)Flushing dad1202300d1ddd3a95c32af1ad98ed0/B: closing flushed file at 1732376443191Flushing dad1202300d1ddd3a95c32af1ad98ed0/C: creating writer at 1732376443212 (+21 ms)Flushing dad1202300d1ddd3a95c32af1ad98ed0/C: appending metadata at 1732376443215 (+3 ms)Flushing dad1202300d1ddd3a95c32af1ad98ed0/C: closing flushed file at 1732376443215Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63ab55b3: reopening flushed file at 1732376443636 (+421 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d8b80b6: reopening flushed file at 1732376443643 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17b820ef: reopening flushed file at 1732376443651 (+8 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for dad1202300d1ddd3a95c32af1ad98ed0 in 903ms, sequenceid=531, compaction requested=true at 1732376443659 (+8 ms)Writing region close event to WAL at 1732376443914 (+255 ms)Running coprocessor post-close hooks at 1732376443923 (+9 ms)Closed at 1732376443923 2024-11-23T15:40:43,927 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:43,927 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=dad1202300d1ddd3a95c32af1ad98ed0, regionState=CLOSED 2024-11-23T15:40:43,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 because future has completed 2024-11-23T15:40:43,933 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-23T15:40:43,933 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure dad1202300d1ddd3a95c32af1ad98ed0, server=b712f9af2c12,46589,1732376409194 in 1.3350 sec 2024-11-23T15:40:43,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-23T15:40:43,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dad1202300d1ddd3a95c32af1ad98ed0, UNASSIGN in 1.3420 sec 2024-11-23T15:40:43,937 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-23T15:40:43,937 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.3490 sec 2024-11-23T15:40:43,938 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376443938"}]},"ts":"1732376443938"} 2024-11-23T15:40:43,940 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T15:40:43,940 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T15:40:43,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3670 sec 2024-11-23T15:40:44,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-23T15:40:44,735 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:40:44,739 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-23T15:40:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:40:44,746 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=35, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:40:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-23T15:40:44,749 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=35, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:40:44,754 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:44,759 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A, FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B, FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C, FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/recovered.edits] 2024-11-23T15:40:44,764 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/21749f1299f54ee38d643e2ebbff8148 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/21749f1299f54ee38d643e2ebbff8148 2024-11-23T15:40:44,767 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/7eda309982e6435c8a5ea41b877d1dc1 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/7eda309982e6435c8a5ea41b877d1dc1 2024-11-23T15:40:44,769 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/99dab134fd24493da8fa70b13b756e0d to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/A/99dab134fd24493da8fa70b13b756e0d 2024-11-23T15:40:44,774 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1b7ecfc625d841c487ce4f996d6b9944 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/1b7ecfc625d841c487ce4f996d6b9944 2024-11-23T15:40:44,777 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/509b318d367c4e59b5529cf579b3563a to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/509b318d367c4e59b5529cf579b3563a 2024-11-23T15:40:44,780 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/5f94f978e0204dde9ec574ea6e3a94b1 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/B/5f94f978e0204dde9ec574ea6e3a94b1 2024-11-23T15:40:44,783 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/771e94afc80e4ecab0f67bdde986f5f7 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/771e94afc80e4ecab0f67bdde986f5f7 2024-11-23T15:40:44,786 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/79cd25aa302342a996b071bfa30d7716 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/79cd25aa302342a996b071bfa30d7716 2024-11-23T15:40:44,791 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/d7c174d0991a4eceacce63bdd25b2e75 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/C/d7c174d0991a4eceacce63bdd25b2e75 2024-11-23T15:40:44,794 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/recovered.edits/534.seqid to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0/recovered.edits/534.seqid 2024-11-23T15:40:44,794 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/dad1202300d1ddd3a95c32af1ad98ed0 2024-11-23T15:40:44,794 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T15:40:44,800 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=35, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:40:44,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-23T15:40:44,809 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T15:40:44,839 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T15:40:44,841 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=35, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:40:44,841 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T15:40:44,841 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732376444841"}]},"ts":"9223372036854775807"} 2024-11-23T15:40:44,847 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T15:40:44,847 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => dad1202300d1ddd3a95c32af1ad98ed0, NAME => 'TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T15:40:44,847 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T15:40:44,848 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732376444848"}]},"ts":"9223372036854775807"} 2024-11-23T15:40:44,851 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T15:40:44,852 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=35, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:40:44,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-23T15:40:44,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 112 msec 2024-11-23T15:40:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-23T15:40:45,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T15:40:45,065 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:40:45,079 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;b712f9af2c12:46589-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1089955776_22 at /127.0.0.1:56860 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=408 (was 322) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7470 (was 8520) 2024-11-23T15:40:45,088 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=408, ProcessCount=11, AvailableMemoryMB=7469 2024-11-23T15:40:45,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T15:40:45,090 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:40:45,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T15:40:45,094 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T15:40:45,094 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:45,095 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T15:40:45,096 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 36 2024-11-23T15:40:45,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-23T15:40:45,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741957_1133 (size=959) 2024-11-23T15:40:45,115 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e 2024-11-23T15:40:45,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741958_1134 (size=53) 2024-11-23T15:40:45,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-23T15:40:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-23T15:40:45,527 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:40:45,527 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e50851c96ec6702e3c34ccb64f6cb9d4, disabling compactions & flushes 2024-11-23T15:40:45,527 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:45,527 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:45,527 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. after waiting 0 ms 2024-11-23T15:40:45,527 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:45,527 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:45,527 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e50851c96ec6702e3c34ccb64f6cb9d4: Waiting for close lock at 1732376445527Disabling compacts and flushes for region at 1732376445527Disabling writes for close at 1732376445527Writing region close event to WAL at 1732376445527Closed at 1732376445527 2024-11-23T15:40:45,529 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T15:40:45,529 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732376445529"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732376445529"}]},"ts":"1732376445529"} 2024-11-23T15:40:45,531 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T15:40:45,535 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T15:40:45,535 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376445535"}]},"ts":"1732376445535"} 2024-11-23T15:40:45,537 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T15:40:45,538 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e50851c96ec6702e3c34ccb64f6cb9d4, ASSIGN}] 2024-11-23T15:40:45,539 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e50851c96ec6702e3c34ccb64f6cb9d4, ASSIGN 2024-11-23T15:40:45,540 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e50851c96ec6702e3c34ccb64f6cb9d4, ASSIGN; state=OFFLINE, location=b712f9af2c12,46589,1732376409194; forceNewPlan=false, retain=false 2024-11-23T15:40:45,691 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=e50851c96ec6702e3c34ccb64f6cb9d4, regionState=OPENING, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:45,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e50851c96ec6702e3c34ccb64f6cb9d4, ASSIGN because future has completed 2024-11-23T15:40:45,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194}] 2024-11-23T15:40:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-23T15:40:45,853 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:45,853 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:40:45,853 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:45,854 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:40:45,854 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:45,854 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:45,856 INFO [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:45,860 INFO [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e50851c96ec6702e3c34ccb64f6cb9d4 columnFamilyName A 2024-11-23T15:40:45,860 DEBUG [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:45,861 INFO [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] regionserver.HStore(327): Store=e50851c96ec6702e3c34ccb64f6cb9d4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:40:45,862 INFO [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:45,863 INFO [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e50851c96ec6702e3c34ccb64f6cb9d4 columnFamilyName B 2024-11-23T15:40:45,863 DEBUG [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:45,864 INFO [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] regionserver.HStore(327): Store=e50851c96ec6702e3c34ccb64f6cb9d4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:40:45,864 INFO [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:45,865 INFO [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e50851c96ec6702e3c34ccb64f6cb9d4 columnFamilyName C 2024-11-23T15:40:45,865 DEBUG [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:45,865 INFO [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] regionserver.HStore(327): Store=e50851c96ec6702e3c34ccb64f6cb9d4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:40:45,866 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:45,866 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:45,867 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:45,868 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:45,868 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:45,868 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T15:40:45,869 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:45,871 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:40:45,871 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened e50851c96ec6702e3c34ccb64f6cb9d4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73485690, jitterRate=0.09502211213111877}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T15:40:45,871 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:45,872 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for e50851c96ec6702e3c34ccb64f6cb9d4: Running coprocessor pre-open hook at 1732376445854Writing region info on filesystem at 1732376445854Initializing all the Stores at 1732376445856 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376445856Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376445856Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376445856Cleaning up temporary data from old regions at 1732376445868 (+12 ms)Running coprocessor post-open hooks at 1732376445871 (+3 ms)Region opened successfully at 1732376445872 (+1 ms) 2024-11-23T15:40:45,873 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., pid=38, masterSystemTime=1732376445846 2024-11-23T15:40:45,875 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:45,875 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:45,876 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=e50851c96ec6702e3c34ccb64f6cb9d4, regionState=OPEN, openSeqNum=2, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:45,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 because future has completed 2024-11-23T15:40:45,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-23T15:40:45,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 in 185 msec 2024-11-23T15:40:45,881 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-23T15:40:45,881 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e50851c96ec6702e3c34ccb64f6cb9d4, ASSIGN in 342 msec 2024-11-23T15:40:45,882 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T15:40:45,882 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376445882"}]},"ts":"1732376445882"} 2024-11-23T15:40:45,884 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T15:40:45,885 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=36, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T15:40:45,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 795 msec 2024-11-23T15:40:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-23T15:40:46,235 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:40:46,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:46,236 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:40:46,237 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:40:46,238 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:40:46,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:40:46,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:40:46,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c45a5be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:46,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:40:46,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:40:46,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:46,241 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:40:46,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f1990d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:46,242 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:40:46,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:40:46,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:40:46,245 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54532, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:40:46,247 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T15:40:46,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is b712f9af2c12,42351,1732376408377 2024-11-23T15:40:46,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@695273e8 2024-11-23T15:40:46,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T15:40:46,250 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T15:40:46,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T15:40:46,256 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:40:46,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-23T15:40:46,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741959_1135 (size=995) 2024-11-23T15:40:46,678 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-23T15:40:46,678 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-23T15:40:46,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T15:40:46,692 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e50851c96ec6702e3c34ccb64f6cb9d4, REOPEN/MOVE}] 2024-11-23T15:40:46,693 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e50851c96ec6702e3c34ccb64f6cb9d4, REOPEN/MOVE 2024-11-23T15:40:46,694 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=e50851c96ec6702e3c34ccb64f6cb9d4, regionState=CLOSING, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:46,695 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e50851c96ec6702e3c34ccb64f6cb9d4, REOPEN/MOVE because future has completed 2024-11-23T15:40:46,696 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T15:40:46,696 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194}] 2024-11-23T15:40:46,849 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(122): Close e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:46,849 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T15:40:46,849 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1722): Closing e50851c96ec6702e3c34ccb64f6cb9d4, disabling compactions & flushes 2024-11-23T15:40:46,849 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:46,849 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:46,849 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. after waiting 0 ms 2024-11-23T15:40:46,849 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:46,854 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-23T15:40:46,855 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:46,855 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegion(1676): Region close journal for e50851c96ec6702e3c34ccb64f6cb9d4: Waiting for close lock at 1732376446849Running coprocessor pre-close hooks at 1732376446849Disabling compacts and flushes for region at 1732376446849Disabling writes for close at 1732376446849Writing region close event to WAL at 1732376446850 (+1 ms)Running coprocessor post-close hooks at 1732376446855 (+5 ms)Closed at 1732376446855 2024-11-23T15:40:46,855 WARN [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] regionserver.HRegionServer(3299): Not adding moved region record: e50851c96ec6702e3c34ccb64f6cb9d4 to self. 2024-11-23T15:40:46,857 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=42}] handler.UnassignRegionHandler(157): Closed e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:46,858 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=e50851c96ec6702e3c34ccb64f6cb9d4, regionState=CLOSED 2024-11-23T15:40:46,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=41, state=RUNNABLE, hasLock=false; CloseRegionProcedure e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 because future has completed 2024-11-23T15:40:46,864 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=41 2024-11-23T15:40:46,864 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=41, state=SUCCESS, hasLock=false; CloseRegionProcedure e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 in 165 msec 2024-11-23T15:40:46,865 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e50851c96ec6702e3c34ccb64f6cb9d4, REOPEN/MOVE; state=CLOSED, location=b712f9af2c12,46589,1732376409194; forceNewPlan=false, retain=true 2024-11-23T15:40:47,015 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=e50851c96ec6702e3c34ccb64f6cb9d4, regionState=OPENING, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:47,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e50851c96ec6702e3c34ccb64f6cb9d4, REOPEN/MOVE because future has completed 2024-11-23T15:40:47,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194}] 2024-11-23T15:40:47,202 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:47,202 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:40:47,203 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:47,203 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:40:47,203 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:47,204 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:47,211 INFO [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:47,220 INFO [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e50851c96ec6702e3c34ccb64f6cb9d4 columnFamilyName A 2024-11-23T15:40:47,223 DEBUG [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:47,223 INFO [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] regionserver.HStore(327): Store=e50851c96ec6702e3c34ccb64f6cb9d4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:40:47,223 INFO [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:47,224 INFO [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e50851c96ec6702e3c34ccb64f6cb9d4 columnFamilyName B 2024-11-23T15:40:47,224 DEBUG [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:47,225 INFO [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] regionserver.HStore(327): Store=e50851c96ec6702e3c34ccb64f6cb9d4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:40:47,225 INFO [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:47,225 INFO [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e50851c96ec6702e3c34ccb64f6cb9d4 columnFamilyName C 2024-11-23T15:40:47,225 DEBUG [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:47,226 INFO [StoreOpener-e50851c96ec6702e3c34ccb64f6cb9d4-1 {}] regionserver.HStore(327): Store=e50851c96ec6702e3c34ccb64f6cb9d4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:40:47,226 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:47,226 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:47,227 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:47,228 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:47,228 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:47,229 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T15:40:47,230 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:47,231 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened e50851c96ec6702e3c34ccb64f6cb9d4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59490283, jitterRate=-0.11352570354938507}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T15:40:47,231 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:47,233 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for e50851c96ec6702e3c34ccb64f6cb9d4: Running coprocessor pre-open hook at 1732376447204Writing region info on filesystem at 1732376447204Initializing all the Stores at 1732376447206 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376447206Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376447206Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376447206Cleaning up temporary data from old regions at 1732376447228 (+22 ms)Running coprocessor post-open hooks at 1732376447231 (+3 ms)Region opened successfully at 1732376447233 (+2 ms) 2024-11-23T15:40:47,234 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., pid=43, masterSystemTime=1732376447195 2024-11-23T15:40:47,236 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:47,236 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:47,237 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=e50851c96ec6702e3c34ccb64f6cb9d4, regionState=OPEN, openSeqNum=5, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:47,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 because future has completed 2024-11-23T15:40:47,240 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=41 2024-11-23T15:40:47,240 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=41, state=SUCCESS, hasLock=false; OpenRegionProcedure e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 in 221 msec 2024-11-23T15:40:47,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-23T15:40:47,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e50851c96ec6702e3c34ccb64f6cb9d4, REOPEN/MOVE in 549 msec 2024-11-23T15:40:47,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-23T15:40:47,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 559 msec 2024-11-23T15:40:47,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 987 msec 2024-11-23T15:40:47,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-23T15:40:47,250 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:40:47,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,251 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:40:47,251 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:40:47,252 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:40:47,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:40:47,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:40:47,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f5ad682, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:40:47,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:40:47,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:47,254 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:40:47,254 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54f321aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,255 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:40:47,255 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:40:47,256 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:40:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:40:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:40:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ba1929, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:40:47,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:40:47,257 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:47,258 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:40:47,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c461885, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,259 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,260 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:40:47,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:40:47,260 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:40:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:40:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:40:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f4c653b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:40:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:40:47,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:47,262 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:40:47,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e5d146, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,264 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:40:47,264 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:40:47,265 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:40:47,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:40:47,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:40:47,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b99a2a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:40:47,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:40:47,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:47,267 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:40:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bbf2daa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,268 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:40:47,268 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:40:47,269 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:40:47,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:40:47,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:40:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ecec32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:40:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:40:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:47,271 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:40:47,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35de9248, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,272 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,272 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:40:47,272 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:40:47,273 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:40:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:40:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:40:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f4b1dfb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:40:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:40:47,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:47,274 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:40:47,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f30d43f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,275 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,275 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:40:47,275 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:40:47,276 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:40:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:40:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:40:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fa6b1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:40:47,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:40:47,277 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:47,277 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:40:47,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7a6519, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,279 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:40:47,279 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:40:47,279 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:40:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:40:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:40:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d661940, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:40:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:40:47,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:47,281 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:40:47,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6739a3af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,282 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:40:47,283 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:40:47,283 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:40:47,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:40:47,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:40:47,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@796f8ea7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:40:47,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:40:47,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:40:47,285 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:40:47,286 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75874e80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:40:47,286 DEBUG [Thread-682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:40:47,286 DEBUG [Thread-675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:40:47,287 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:40:47,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:40:47,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:40:47,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:40:47,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:40:47,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-23T15:40:47,289 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:40:47,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T15:40:47,289 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:40:47,289 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:40:47,290 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:40:47,290 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:40:47,290 DEBUG [Thread-688 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:40:47,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:40:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:40:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:40:47,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:40:47,293 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:40:47,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:40:47,295 DEBUG [Thread-673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:40:47,296 DEBUG [Thread-684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:40:47,296 DEBUG [Thread-686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:40:47,296 DEBUG [Thread-679 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:40:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:40:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:40:47,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:40:47,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:40:47,297 DEBUG [Thread-677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:40:47,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:40:47,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:40:47,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:40:47,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:40:47,300 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:40:47,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:40:47,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:40:47,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:40:47,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:40:47,301 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:40:47,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:40:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:40:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:40:47,303 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:40:47,304 DEBUG [Thread-671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:40:47,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:40:47,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:40:47,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:40:47,315 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:40:47,317 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:40:47,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:47,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:40:47,320 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:40:47,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e50851c96ec6702e3c34ccb64f6cb9d4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T15:40:47,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123a144ba639d124d10ba713f9dcb4be15f_e50851c96ec6702e3c34ccb64f6cb9d4 is 50, key is test_row_0/A:col10/1732376447313/Put/seqid=0 2024-11-23T15:40:47,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:47,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376507373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:47,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:47,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376507376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:47,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:47,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376507378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:47,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:47,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:47,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376507380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376507380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:47,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:47,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:47,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741960_1136 (size=17034) 2024-11-23T15:40:47,393 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:47,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T15:40:47,399 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123a144ba639d124d10ba713f9dcb4be15f_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123a144ba639d124d10ba713f9dcb4be15f_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:47,400 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/bff9ef12b9144589921021f713701dc9, store: [table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:47,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/bff9ef12b9144589921021f713701dc9 is 175, key is test_row_0/A:col10/1732376447313/Put/seqid=0 2024-11-23T15:40:47,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741961_1137 (size=48139) 2024-11-23T15:40:47,421 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/bff9ef12b9144589921021f713701dc9 2024-11-23T15:40:47,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T15:40:47,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:47,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:40:47,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:47,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:47,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:47,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/0acd431422014fb0877f95a9abaa8d86 is 50, key is test_row_0/B:col10/1732376447313/Put/seqid=0 2024-11-23T15:40:47,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376507484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:47,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741962_1138 (size=12001) 2024-11-23T15:40:47,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/0acd431422014fb0877f95a9abaa8d86 2024-11-23T15:40:47,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376507495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:47,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:47,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:47,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376507496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:47,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:47,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:47,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376507496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:47,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376507496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:47,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/14ef2cc565aa484595f924054f9c0950 is 50, key is test_row_0/C:col10/1732376447313/Put/seqid=0 2024-11-23T15:40:47,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741963_1139 (size=12001) 2024-11-23T15:40:47,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/14ef2cc565aa484595f924054f9c0950 2024-11-23T15:40:47,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/bff9ef12b9144589921021f713701dc9 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/bff9ef12b9144589921021f713701dc9 2024-11-23T15:40:47,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/bff9ef12b9144589921021f713701dc9, entries=250, sequenceid=16, filesize=47.0 K 2024-11-23T15:40:47,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/0acd431422014fb0877f95a9abaa8d86 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/0acd431422014fb0877f95a9abaa8d86 2024-11-23T15:40:47,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/0acd431422014fb0877f95a9abaa8d86, entries=150, sequenceid=16, filesize=11.7 K 2024-11-23T15:40:47,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/14ef2cc565aa484595f924054f9c0950 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/14ef2cc565aa484595f924054f9c0950 2024-11-23T15:40:47,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/14ef2cc565aa484595f924054f9c0950, entries=150, sequenceid=16, filesize=11.7 K 2024-11-23T15:40:47,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e50851c96ec6702e3c34ccb64f6cb9d4 in 262ms, sequenceid=16, compaction requested=false 2024-11-23T15:40:47,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:47,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T15:40:47,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:47,600 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing e50851c96ec6702e3c34ccb64f6cb9d4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T15:40:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T15:40:47,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123d6f8270189ba4f479277e0287c617466_e50851c96ec6702e3c34ccb64f6cb9d4 is 50, key is test_row_0/A:col10/1732376447375/Put/seqid=0 2024-11-23T15:40:47,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741964_1140 (size=12154) 2024-11-23T15:40:47,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:47,660 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123d6f8270189ba4f479277e0287c617466_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123d6f8270189ba4f479277e0287c617466_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:47,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/ead4bc2f89164c78adab6f60b173986d, store: [table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:47,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/ead4bc2f89164c78adab6f60b173986d is 175, key is test_row_0/A:col10/1732376447375/Put/seqid=0 2024-11-23T15:40:47,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741965_1141 (size=30955) 2024-11-23T15:40:47,681 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/ead4bc2f89164c78adab6f60b173986d 2024-11-23T15:40:47,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/d0a640ef2a3e41258109cc4fd35f1e87 is 50, key is test_row_0/B:col10/1732376447375/Put/seqid=0 2024-11-23T15:40:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:47,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:40:47,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741966_1142 (size=12001) 2024-11-23T15:40:47,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376507719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:47,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:47,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376507721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:47,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:47,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:47,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376507723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:47,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376507723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:47,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376507723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:47,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:47,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:47,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:47,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376507834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:47,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:47,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376507845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:47,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:47,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376507845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:47,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376507845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:47,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:47,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376507845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T15:40:48,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376508044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376508054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376508054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376508054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376508054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,112 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/d0a640ef2a3e41258109cc4fd35f1e87 2024-11-23T15:40:48,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/ce54fc44f3824f8c97bfd381c3cb1dcd is 50, key is test_row_0/C:col10/1732376447375/Put/seqid=0 2024-11-23T15:40:48,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741967_1143 (size=12001) 2024-11-23T15:40:48,151 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/ce54fc44f3824f8c97bfd381c3cb1dcd 2024-11-23T15:40:48,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/ead4bc2f89164c78adab6f60b173986d as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ead4bc2f89164c78adab6f60b173986d 2024-11-23T15:40:48,169 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ead4bc2f89164c78adab6f60b173986d, entries=150, sequenceid=40, filesize=30.2 K 2024-11-23T15:40:48,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/d0a640ef2a3e41258109cc4fd35f1e87 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/d0a640ef2a3e41258109cc4fd35f1e87 2024-11-23T15:40:48,181 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/d0a640ef2a3e41258109cc4fd35f1e87, entries=150, sequenceid=40, filesize=11.7 K 2024-11-23T15:40:48,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/ce54fc44f3824f8c97bfd381c3cb1dcd as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/ce54fc44f3824f8c97bfd381c3cb1dcd 2024-11-23T15:40:48,193 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/ce54fc44f3824f8c97bfd381c3cb1dcd, entries=150, sequenceid=40, filesize=11.7 K 2024-11-23T15:40:48,194 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e50851c96ec6702e3c34ccb64f6cb9d4 in 594ms, sequenceid=40, compaction requested=false 2024-11-23T15:40:48,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:48,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:48,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-23T15:40:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-23T15:40:48,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-23T15:40:48,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 906 msec 2024-11-23T15:40:48,200 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 912 msec 2024-11-23T15:40:48,358 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:40:48,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e50851c96ec6702e3c34ccb64f6cb9d4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-23T15:40:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:48,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123deb0efc043374943a4ad4432172e3a4c_e50851c96ec6702e3c34ccb64f6cb9d4 is 50, key is test_row_0/A:col10/1732376447720/Put/seqid=0 2024-11-23T15:40:48,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741968_1144 (size=14594) 2024-11-23T15:40:48,389 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:48,397 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123deb0efc043374943a4ad4432172e3a4c_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123deb0efc043374943a4ad4432172e3a4c_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:48,399 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/566e1e6c33234f09b9c7afe5a2294bd8, store: [table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:48,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/566e1e6c33234f09b9c7afe5a2294bd8 is 175, key is test_row_0/A:col10/1732376447720/Put/seqid=0 2024-11-23T15:40:48,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376508400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376508401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376508403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376508404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376508405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741969_1145 (size=39549) 2024-11-23T15:40:48,424 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/566e1e6c33234f09b9c7afe5a2294bd8 2024-11-23T15:40:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T15:40:48,425 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:40:48,426 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:40:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-23T15:40:48,428 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:40:48,430 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:40:48,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:40:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T15:40:48,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/2bad4fda470341b1a2bb5ccbc8d20e2f is 50, key is test_row_0/B:col10/1732376447720/Put/seqid=0 2024-11-23T15:40:48,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741970_1146 (size=12001) 2024-11-23T15:40:48,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/2bad4fda470341b1a2bb5ccbc8d20e2f 2024-11-23T15:40:48,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/8f14a41a2a484c2a8e00bd55b3d62c00 is 50, key is test_row_0/C:col10/1732376447720/Put/seqid=0 2024-11-23T15:40:48,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741971_1147 (size=12001) 2024-11-23T15:40:48,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/8f14a41a2a484c2a8e00bd55b3d62c00 2024-11-23T15:40:48,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/566e1e6c33234f09b9c7afe5a2294bd8 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/566e1e6c33234f09b9c7afe5a2294bd8 2024-11-23T15:40:48,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/566e1e6c33234f09b9c7afe5a2294bd8, entries=200, sequenceid=53, filesize=38.6 K 2024-11-23T15:40:48,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/2bad4fda470341b1a2bb5ccbc8d20e2f as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/2bad4fda470341b1a2bb5ccbc8d20e2f 2024-11-23T15:40:48,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376508514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376508514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376508514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376508514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376508515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/2bad4fda470341b1a2bb5ccbc8d20e2f, entries=150, sequenceid=53, filesize=11.7 K 2024-11-23T15:40:48,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/8f14a41a2a484c2a8e00bd55b3d62c00 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/8f14a41a2a484c2a8e00bd55b3d62c00 2024-11-23T15:40:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T15:40:48,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/8f14a41a2a484c2a8e00bd55b3d62c00, entries=150, sequenceid=53, filesize=11.7 K 2024-11-23T15:40:48,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e50851c96ec6702e3c34ccb64f6cb9d4 in 185ms, sequenceid=53, compaction requested=true 2024-11-23T15:40:48,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:48,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e50851c96ec6702e3c34ccb64f6cb9d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:40:48,543 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:48,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:48,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e50851c96ec6702e3c34ccb64f6cb9d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:40:48,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:48,543 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:48,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e50851c96ec6702e3c34ccb64f6cb9d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:40:48,543 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:48,544 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:48,544 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:48,545 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): e50851c96ec6702e3c34ccb64f6cb9d4/B is initiating minor compaction (all files) 2024-11-23T15:40:48,545 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): e50851c96ec6702e3c34ccb64f6cb9d4/A is initiating minor compaction (all files) 2024-11-23T15:40:48,545 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e50851c96ec6702e3c34ccb64f6cb9d4/A in TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:48,545 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e50851c96ec6702e3c34ccb64f6cb9d4/B in TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:48,545 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/0acd431422014fb0877f95a9abaa8d86, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/d0a640ef2a3e41258109cc4fd35f1e87, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/2bad4fda470341b1a2bb5ccbc8d20e2f] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp, totalSize=35.2 K 2024-11-23T15:40:48,545 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/bff9ef12b9144589921021f713701dc9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ead4bc2f89164c78adab6f60b173986d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/566e1e6c33234f09b9c7afe5a2294bd8] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp, totalSize=115.9 K 2024-11-23T15:40:48,545 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:48,545 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. files: [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/bff9ef12b9144589921021f713701dc9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ead4bc2f89164c78adab6f60b173986d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/566e1e6c33234f09b9c7afe5a2294bd8] 2024-11-23T15:40:48,546 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting bff9ef12b9144589921021f713701dc9, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732376447306 2024-11-23T15:40:48,546 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 0acd431422014fb0877f95a9abaa8d86, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732376447313 2024-11-23T15:40:48,546 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ead4bc2f89164c78adab6f60b173986d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732376447358 2024-11-23T15:40:48,546 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting d0a640ef2a3e41258109cc4fd35f1e87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732376447358 2024-11-23T15:40:48,546 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 566e1e6c33234f09b9c7afe5a2294bd8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732376447716 2024-11-23T15:40:48,546 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bad4fda470341b1a2bb5ccbc8d20e2f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732376447720 2024-11-23T15:40:48,560 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:48,572 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123c65d56d42d524dd3aaf88372b9d561d7_e50851c96ec6702e3c34ccb64f6cb9d4 store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:48,579 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e50851c96ec6702e3c34ccb64f6cb9d4#B#compaction#130 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:48,580 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/b651d8c1ed0647389e2c5f04cc3c94bc is 50, key is test_row_0/B:col10/1732376447720/Put/seqid=0 2024-11-23T15:40:48,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T15:40:48,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:48,583 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing e50851c96ec6702e3c34ccb64f6cb9d4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T15:40:48,588 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123c65d56d42d524dd3aaf88372b9d561d7_e50851c96ec6702e3c34ccb64f6cb9d4, store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:48,588 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c65d56d42d524dd3aaf88372b9d561d7_e50851c96ec6702e3c34ccb64f6cb9d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:48,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123351b52f528374a1c80177f185d44f630_e50851c96ec6702e3c34ccb64f6cb9d4 is 50, key is test_row_0/A:col10/1732376448398/Put/seqid=0 2024-11-23T15:40:48,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741972_1148 (size=12104) 2024-11-23T15:40:48,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741973_1149 (size=4469) 2024-11-23T15:40:48,618 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e50851c96ec6702e3c34ccb64f6cb9d4#A#compaction#129 average throughput is 0.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:48,618 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/e2282c8c965a4159b74ea73534699c8c is 175, key is test_row_0/A:col10/1732376447720/Put/seqid=0 2024-11-23T15:40:48,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741974_1150 (size=12154) 2024-11-23T15:40:48,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741975_1151 (size=31058) 2024-11-23T15:40:48,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:48,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:40:48,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376508736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376508738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376508738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376508739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T15:40:48,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376508739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376508854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376508854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376508854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,857 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376508854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:48,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376508855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:48,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:48,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:49,021 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/b651d8c1ed0647389e2c5f04cc3c94bc as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/b651d8c1ed0647389e2c5f04cc3c94bc 2024-11-23T15:40:49,031 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e50851c96ec6702e3c34ccb64f6cb9d4/B of e50851c96ec6702e3c34ccb64f6cb9d4 into b651d8c1ed0647389e2c5f04cc3c94bc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:49,031 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:49,031 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., storeName=e50851c96ec6702e3c34ccb64f6cb9d4/B, priority=13, startTime=1732376448543; duration=0sec 2024-11-23T15:40:49,031 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:49,031 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e50851c96ec6702e3c34ccb64f6cb9d4:B 2024-11-23T15:40:49,031 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:49,034 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:49,034 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): e50851c96ec6702e3c34ccb64f6cb9d4/C is initiating minor compaction (all files) 2024-11-23T15:40:49,034 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e50851c96ec6702e3c34ccb64f6cb9d4/C in TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:49,038 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/14ef2cc565aa484595f924054f9c0950, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/ce54fc44f3824f8c97bfd381c3cb1dcd, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/8f14a41a2a484c2a8e00bd55b3d62c00] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp, totalSize=35.2 K 2024-11-23T15:40:49,040 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 14ef2cc565aa484595f924054f9c0950, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732376447313 2024-11-23T15:40:49,040 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting ce54fc44f3824f8c97bfd381c3cb1dcd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732376447358 2024-11-23T15:40:49,041 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f14a41a2a484c2a8e00bd55b3d62c00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732376447720 2024-11-23T15:40:49,048 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e50851c96ec6702e3c34ccb64f6cb9d4#C#compaction#132 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:49,049 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/7514975ad0234805af29366434160f45 is 50, key is test_row_0/C:col10/1732376447720/Put/seqid=0 2024-11-23T15:40:49,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T15:40:49,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:49,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376509064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:49,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,066 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:49,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:49,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376509064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:49,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:49,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:49,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376509064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:49,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376509065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,069 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123351b52f528374a1c80177f185d44f630_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123351b52f528374a1c80177f185d44f630_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:49,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:49,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:49,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376509065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:49,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:49,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/ccb85e09e6a748dd9b42f2a437eb34fb, store: [table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:49,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/ccb85e09e6a748dd9b42f2a437eb34fb is 175, key is test_row_0/A:col10/1732376448398/Put/seqid=0 2024-11-23T15:40:49,077 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/e2282c8c965a4159b74ea73534699c8c as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/e2282c8c965a4159b74ea73534699c8c 2024-11-23T15:40:49,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741976_1152 (size=12104) 2024-11-23T15:40:49,085 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e50851c96ec6702e3c34ccb64f6cb9d4/A of e50851c96ec6702e3c34ccb64f6cb9d4 into e2282c8c965a4159b74ea73534699c8c(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:49,085 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:49,085 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., storeName=e50851c96ec6702e3c34ccb64f6cb9d4/A, priority=13, startTime=1732376448543; duration=0sec 2024-11-23T15:40:49,086 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:49,086 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e50851c96ec6702e3c34ccb64f6cb9d4:A 2024-11-23T15:40:49,087 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/7514975ad0234805af29366434160f45 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/7514975ad0234805af29366434160f45 2024-11-23T15:40:49,097 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e50851c96ec6702e3c34ccb64f6cb9d4/C of e50851c96ec6702e3c34ccb64f6cb9d4 into 7514975ad0234805af29366434160f45(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:49,097 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:49,097 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., storeName=e50851c96ec6702e3c34ccb64f6cb9d4/C, priority=13, startTime=1732376448543; duration=0sec 2024-11-23T15:40:49,097 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:49,097 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e50851c96ec6702e3c34ccb64f6cb9d4:C 2024-11-23T15:40:49,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741977_1153 (size=30955) 2024-11-23T15:40:49,101 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/ccb85e09e6a748dd9b42f2a437eb34fb 2024-11-23T15:40:49,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/2dad067be64842d3b103442711ef62aa is 50, key is test_row_0/B:col10/1732376448398/Put/seqid=0 2024-11-23T15:40:49,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741978_1154 (size=12001) 2024-11-23T15:40:49,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:49,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376509375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:49,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:49,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376509376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:49,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:49,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376509376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:49,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:49,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376509376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:49,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:49,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376509376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:49,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:49,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:49,455 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T15:40:49,531 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/2dad067be64842d3b103442711ef62aa 2024-11-23T15:40:49,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/6122f533d307475ba416a422219077e4 is 50, key is test_row_0/C:col10/1732376448398/Put/seqid=0 2024-11-23T15:40:49,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741979_1155 (size=12001) 2024-11-23T15:40:49,551 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/6122f533d307475ba416a422219077e4 2024-11-23T15:40:49,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/ccb85e09e6a748dd9b42f2a437eb34fb as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ccb85e09e6a748dd9b42f2a437eb34fb 2024-11-23T15:40:49,564 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ccb85e09e6a748dd9b42f2a437eb34fb, entries=150, sequenceid=76, filesize=30.2 K 2024-11-23T15:40:49,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T15:40:49,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/2dad067be64842d3b103442711ef62aa as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/2dad067be64842d3b103442711ef62aa 2024-11-23T15:40:49,582 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/2dad067be64842d3b103442711ef62aa, entries=150, sequenceid=76, filesize=11.7 K 2024-11-23T15:40:49,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/6122f533d307475ba416a422219077e4 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/6122f533d307475ba416a422219077e4 2024-11-23T15:40:49,591 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/6122f533d307475ba416a422219077e4, entries=150, sequenceid=76, filesize=11.7 K 2024-11-23T15:40:49,595 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for e50851c96ec6702e3c34ccb64f6cb9d4 in 1012ms, sequenceid=76, compaction requested=false 2024-11-23T15:40:49,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:49,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:49,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-23T15:40:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-23T15:40:49,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-23T15:40:49,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1670 sec 2024-11-23T15:40:49,605 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.1780 sec 2024-11-23T15:40:49,891 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:40:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:49,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e50851c96ec6702e3c34ccb64f6cb9d4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-23T15:40:49,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c7a69efe73c3473ca20747c9bac88040_e50851c96ec6702e3c34ccb64f6cb9d4 is 50, key is test_row_0/A:col10/1732376448738/Put/seqid=0 2024-11-23T15:40:49,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376509919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:49,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:49,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376509925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:49,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376509926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:49,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376509930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:49,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:49,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376509931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:49,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:49,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:49,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741980_1156 (size=12154) 2024-11-23T15:40:50,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:50,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376510034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:50,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376510034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:50,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376510035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:50,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,036 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:50,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376510044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:50,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:50,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376510044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,047 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:50,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:50,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376510244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:50,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376510244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:50,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376510244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:50,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:50,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376510255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:50,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376510255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:50,347 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:50,362 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c7a69efe73c3473ca20747c9bac88040_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c7a69efe73c3473ca20747c9bac88040_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:50,363 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/0adaaee5fd014407ba3a87a3e48e5dd4, store: [table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:50,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/0adaaee5fd014407ba3a87a3e48e5dd4 is 175, key is test_row_0/A:col10/1732376448738/Put/seqid=0 2024-11-23T15:40:50,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741981_1157 (size=30955) 2024-11-23T15:40:50,413 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/0adaaee5fd014407ba3a87a3e48e5dd4 2024-11-23T15:40:50,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/f77b6a7c93da4948b408c98976dff510 is 50, key is test_row_0/B:col10/1732376448738/Put/seqid=0 2024-11-23T15:40:50,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741982_1158 (size=12001) 2024-11-23T15:40:50,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/f77b6a7c93da4948b408c98976dff510 2024-11-23T15:40:50,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/f47d234622e64f6ab9b1109e0af38e04 is 50, key is test_row_0/C:col10/1732376448738/Put/seqid=0 2024-11-23T15:40:50,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:50,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376510554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:50,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:50,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:50,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376510554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:50,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:50,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:50,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376510554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:50,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:50,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376510565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:50,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,573 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:50,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:50,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376510565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:50,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:50,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741983_1159 (size=12001) 2024-11-23T15:40:50,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/f47d234622e64f6ab9b1109e0af38e04 2024-11-23T15:40:50,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T15:40:50,585 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:40:50,585 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:40:50,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-23T15:40:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T15:40:50,588 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:40:50,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/0adaaee5fd014407ba3a87a3e48e5dd4 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/0adaaee5fd014407ba3a87a3e48e5dd4 2024-11-23T15:40:50,591 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:40:50,591 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:40:50,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/0adaaee5fd014407ba3a87a3e48e5dd4, entries=150, sequenceid=95, filesize=30.2 K 2024-11-23T15:40:50,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/f77b6a7c93da4948b408c98976dff510 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/f77b6a7c93da4948b408c98976dff510 2024-11-23T15:40:50,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/f77b6a7c93da4948b408c98976dff510, entries=150, sequenceid=95, filesize=11.7 K 2024-11-23T15:40:50,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/f47d234622e64f6ab9b1109e0af38e04 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/f47d234622e64f6ab9b1109e0af38e04 2024-11-23T15:40:50,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/f47d234622e64f6ab9b1109e0af38e04, entries=150, sequenceid=95, filesize=11.7 K 2024-11-23T15:40:50,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for e50851c96ec6702e3c34ccb64f6cb9d4 in 734ms, sequenceid=95, compaction requested=true 2024-11-23T15:40:50,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:50,627 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:50,629 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:50,629 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): e50851c96ec6702e3c34ccb64f6cb9d4/A is initiating minor compaction (all files) 2024-11-23T15:40:50,629 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e50851c96ec6702e3c34ccb64f6cb9d4/A in TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:50,629 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/e2282c8c965a4159b74ea73534699c8c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ccb85e09e6a748dd9b42f2a437eb34fb, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/0adaaee5fd014407ba3a87a3e48e5dd4] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp, totalSize=90.8 K 2024-11-23T15:40:50,629 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:50,629 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. files: [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/e2282c8c965a4159b74ea73534699c8c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ccb85e09e6a748dd9b42f2a437eb34fb, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/0adaaee5fd014407ba3a87a3e48e5dd4] 2024-11-23T15:40:50,630 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2282c8c965a4159b74ea73534699c8c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732376447720 2024-11-23T15:40:50,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e50851c96ec6702e3c34ccb64f6cb9d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:40:50,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:50,631 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:50,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e50851c96ec6702e3c34ccb64f6cb9d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:40:50,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:50,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e50851c96ec6702e3c34ccb64f6cb9d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:40:50,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:50,639 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccb85e09e6a748dd9b42f2a437eb34fb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732376448398 2024-11-23T15:40:50,640 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:50,640 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): e50851c96ec6702e3c34ccb64f6cb9d4/B is initiating minor compaction (all files) 2024-11-23T15:40:50,640 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e50851c96ec6702e3c34ccb64f6cb9d4/B in TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:50,641 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/b651d8c1ed0647389e2c5f04cc3c94bc, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/2dad067be64842d3b103442711ef62aa, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/f77b6a7c93da4948b408c98976dff510] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp, totalSize=35.3 K 2024-11-23T15:40:50,641 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting b651d8c1ed0647389e2c5f04cc3c94bc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732376447720 2024-11-23T15:40:50,642 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0adaaee5fd014407ba3a87a3e48e5dd4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732376448737 2024-11-23T15:40:50,642 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dad067be64842d3b103442711ef62aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732376448398 2024-11-23T15:40:50,643 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting f77b6a7c93da4948b408c98976dff510, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732376448737 2024-11-23T15:40:50,669 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e50851c96ec6702e3c34ccb64f6cb9d4#B#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:50,669 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/35c532aa098d43fa80fa09c73f52f893 is 50, key is test_row_0/B:col10/1732376448738/Put/seqid=0 2024-11-23T15:40:50,692 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:50,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T15:40:50,707 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123ff7b6125919a44398d428bece4917e1e_e50851c96ec6702e3c34ccb64f6cb9d4 store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:50,711 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123ff7b6125919a44398d428bece4917e1e_e50851c96ec6702e3c34ccb64f6cb9d4, store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:50,711 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ff7b6125919a44398d428bece4917e1e_e50851c96ec6702e3c34ccb64f6cb9d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:50,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741984_1160 (size=12207) 2024-11-23T15:40:50,736 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/35c532aa098d43fa80fa09c73f52f893 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/35c532aa098d43fa80fa09c73f52f893 2024-11-23T15:40:50,743 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e50851c96ec6702e3c34ccb64f6cb9d4/B of e50851c96ec6702e3c34ccb64f6cb9d4 into 35c532aa098d43fa80fa09c73f52f893(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:50,743 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:50,743 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., storeName=e50851c96ec6702e3c34ccb64f6cb9d4/B, priority=13, startTime=1732376450631; duration=0sec 2024-11-23T15:40:50,743 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:50,743 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e50851c96ec6702e3c34ccb64f6cb9d4:B 2024-11-23T15:40:50,743 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:50,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-23T15:40:50,745 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:50,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:50,745 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): e50851c96ec6702e3c34ccb64f6cb9d4/C is initiating minor compaction (all files) 2024-11-23T15:40:50,745 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e50851c96ec6702e3c34ccb64f6cb9d4/C in TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:50,745 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing e50851c96ec6702e3c34ccb64f6cb9d4 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-23T15:40:50,745 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/7514975ad0234805af29366434160f45, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/6122f533d307475ba416a422219077e4, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/f47d234622e64f6ab9b1109e0af38e04] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp, totalSize=35.3 K 2024-11-23T15:40:50,746 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 7514975ad0234805af29366434160f45, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732376447720 2024-11-23T15:40:50,749 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6122f533d307475ba416a422219077e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732376448398 2024-11-23T15:40:50,750 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting f47d234622e64f6ab9b1109e0af38e04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732376448737 2024-11-23T15:40:50,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741985_1161 (size=4469) 2024-11-23T15:40:50,778 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e50851c96ec6702e3c34ccb64f6cb9d4#A#compaction#139 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:50,778 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/028cb4a96bb54af48231f7e92758b467 is 175, key is test_row_0/A:col10/1732376448738/Put/seqid=0 2024-11-23T15:40:50,801 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e50851c96ec6702e3c34ccb64f6cb9d4#C#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:50,801 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/8be98f63542a4c1c9032cca088fbe977 is 50, key is test_row_0/C:col10/1732376448738/Put/seqid=0 2024-11-23T15:40:50,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c021a6ea36fd4344818650456c0e2561_e50851c96ec6702e3c34ccb64f6cb9d4 is 50, key is test_row_0/A:col10/1732376449916/Put/seqid=0 2024-11-23T15:40:50,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741986_1162 (size=31161) 2024-11-23T15:40:50,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741987_1163 (size=12207) 2024-11-23T15:40:50,882 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/8be98f63542a4c1c9032cca088fbe977 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/8be98f63542a4c1c9032cca088fbe977 2024-11-23T15:40:50,891 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e50851c96ec6702e3c34ccb64f6cb9d4/C of e50851c96ec6702e3c34ccb64f6cb9d4 into 8be98f63542a4c1c9032cca088fbe977(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:50,891 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:50,891 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., storeName=e50851c96ec6702e3c34ccb64f6cb9d4/C, priority=13, startTime=1732376450631; duration=0sec 2024-11-23T15:40:50,891 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:50,892 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e50851c96ec6702e3c34ccb64f6cb9d4:C 2024-11-23T15:40:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T15:40:50,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741988_1164 (size=12154) 2024-11-23T15:40:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:51,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:40:51,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376511096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:51,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:51,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376511097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:51,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376511098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:51,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376511095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:51,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376511110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T15:40:51,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376511224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376511224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:51,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376511224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376511224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,230 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:51,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376511224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:51,267 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/028cb4a96bb54af48231f7e92758b467 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/028cb4a96bb54af48231f7e92758b467 2024-11-23T15:40:51,275 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e50851c96ec6702e3c34ccb64f6cb9d4/A of e50851c96ec6702e3c34ccb64f6cb9d4 into 028cb4a96bb54af48231f7e92758b467(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:51,275 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:51,275 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., storeName=e50851c96ec6702e3c34ccb64f6cb9d4/A, priority=13, startTime=1732376450627; duration=0sec 2024-11-23T15:40:51,275 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:51,275 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e50851c96ec6702e3c34ccb64f6cb9d4:A 2024-11-23T15:40:51,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:51,316 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c021a6ea36fd4344818650456c0e2561_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c021a6ea36fd4344818650456c0e2561_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:51,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/6c5e18df32aa485bbfd8e8d4ab8b3ac3, store: [table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:51,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/6c5e18df32aa485bbfd8e8d4ab8b3ac3 is 175, key is test_row_0/A:col10/1732376449916/Put/seqid=0 2024-11-23T15:40:51,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741989_1165 (size=30955) 2024-11-23T15:40:51,325 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/6c5e18df32aa485bbfd8e8d4ab8b3ac3 2024-11-23T15:40:51,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/d27b474a3ed34f80aa3b584344d9cb23 is 50, key is test_row_0/B:col10/1732376449916/Put/seqid=0 2024-11-23T15:40:51,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741990_1166 (size=12001) 2024-11-23T15:40:51,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376511434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:51,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376511435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:51,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376511435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:51,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376511435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:51,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376511435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:51,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:51,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T15:40:51,742 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/d27b474a3ed34f80aa3b584344d9cb23 2024-11-23T15:40:51,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376511745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:51,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376511745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:51,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376511754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:51,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:51,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376511754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:51,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376511754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:51,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:51,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/0fa4cbf7e230457cba0c355d69556208 is 50, key is test_row_0/C:col10/1732376449916/Put/seqid=0 2024-11-23T15:40:51,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741991_1167 (size=12001) 2024-11-23T15:40:52,212 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/0fa4cbf7e230457cba0c355d69556208 2024-11-23T15:40:52,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/6c5e18df32aa485bbfd8e8d4ab8b3ac3 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/6c5e18df32aa485bbfd8e8d4ab8b3ac3 2024-11-23T15:40:52,225 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/6c5e18df32aa485bbfd8e8d4ab8b3ac3, entries=150, sequenceid=116, filesize=30.2 K 2024-11-23T15:40:52,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/d27b474a3ed34f80aa3b584344d9cb23 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/d27b474a3ed34f80aa3b584344d9cb23 2024-11-23T15:40:52,234 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/d27b474a3ed34f80aa3b584344d9cb23, entries=150, sequenceid=116, filesize=11.7 K 2024-11-23T15:40:52,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/0fa4cbf7e230457cba0c355d69556208 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/0fa4cbf7e230457cba0c355d69556208 2024-11-23T15:40:52,250 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/0fa4cbf7e230457cba0c355d69556208, entries=150, sequenceid=116, filesize=11.7 K 2024-11-23T15:40:52,251 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for e50851c96ec6702e3c34ccb64f6cb9d4 in 1506ms, sequenceid=116, compaction requested=false 2024-11-23T15:40:52,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:52,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:52,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-23T15:40:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-23T15:40:52,256 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-23T15:40:52,256 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6620 sec 2024-11-23T15:40:52,259 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.6710 sec 2024-11-23T15:40:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:52,268 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:40:52,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e50851c96ec6702e3c34ccb64f6cb9d4 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-23T15:40:52,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411235c7519f595764664b79673b46f27e1f1_e50851c96ec6702e3c34ccb64f6cb9d4 is 50, key is test_row_0/A:col10/1732376451098/Put/seqid=0 2024-11-23T15:40:52,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376512295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376512297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:52,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:52,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376512299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376512299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:52,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376512300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:52,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741992_1168 (size=17284) 2024-11-23T15:40:52,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376512414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:52,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376512415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:52,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:52,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:52,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376512415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376512415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376512415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:52,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376512624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:52,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376512624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:52,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376512624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:52,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376512624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:52,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:52,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376512624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:52,708 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:52,713 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411235c7519f595764664b79673b46f27e1f1_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235c7519f595764664b79673b46f27e1f1_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:52,714 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/e62c6e1f745b4d40b191cb775b99f80a, store: [table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:52,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/e62c6e1f745b4d40b191cb775b99f80a is 175, key is test_row_0/A:col10/1732376451098/Put/seqid=0 2024-11-23T15:40:52,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741993_1169 (size=48389) 2024-11-23T15:40:52,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T15:40:52,735 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:40:52,735 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:40:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-23T15:40:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T15:40:52,736 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:40:52,737 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:40:52,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:40:52,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T15:40:52,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-23T15:40:52,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:52,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:40:52,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:52,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:52,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:52,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:52,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:52,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376512934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376512934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:52,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376512934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:52,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376512934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:52,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376512935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:52,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:53,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-23T15:40:53,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:53,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:40:53,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:53,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:53,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:53,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:53,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T15:40:53,121 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/e62c6e1f745b4d40b191cb775b99f80a 2024-11-23T15:40:53,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/dc226e0965474e108d7f25c4feeee487 is 50, key is test_row_0/B:col10/1732376451098/Put/seqid=0 2024-11-23T15:40:53,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741994_1170 (size=12151) 2024-11-23T15:40:53,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-23T15:40:53,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:53,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:40:53,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:53,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:53,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:53,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:53,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-23T15:40:53,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:53,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:40:53,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:53,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:53,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:53,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:53,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T15:40:53,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:53,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376513444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:53,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:53,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:53,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:53,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:53,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376513445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:53,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:53,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:53,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376513445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:53,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:53,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:53,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:53,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:53,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376513445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:53,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:53,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:53,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:53,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:53,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376513455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:53,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:53,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:53,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:53,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-23T15:40:53,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:53,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:40:53,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:53,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:53,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:53,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:53,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/dc226e0965474e108d7f25c4feeee487 2024-11-23T15:40:53,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/6d14f2df26a54bff8014fc23cf527dff is 50, key is test_row_0/C:col10/1732376451098/Put/seqid=0 2024-11-23T15:40:53,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741995_1171 (size=12151) 2024-11-23T15:40:53,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/6d14f2df26a54bff8014fc23cf527dff 2024-11-23T15:40:53,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/e62c6e1f745b4d40b191cb775b99f80a as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/e62c6e1f745b4d40b191cb775b99f80a 2024-11-23T15:40:53,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/e62c6e1f745b4d40b191cb775b99f80a, entries=250, sequenceid=136, filesize=47.3 K 2024-11-23T15:40:53,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/dc226e0965474e108d7f25c4feeee487 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/dc226e0965474e108d7f25c4feeee487 2024-11-23T15:40:53,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-23T15:40:53,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:53,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:40:53,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:53,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:53,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:53,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/dc226e0965474e108d7f25c4feeee487, entries=150, sequenceid=136, filesize=11.9 K 2024-11-23T15:40:53,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/6d14f2df26a54bff8014fc23cf527dff as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/6d14f2df26a54bff8014fc23cf527dff 2024-11-23T15:40:53,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/6d14f2df26a54bff8014fc23cf527dff, entries=150, sequenceid=136, filesize=11.9 K 2024-11-23T15:40:53,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e50851c96ec6702e3c34ccb64f6cb9d4 in 1410ms, sequenceid=136, compaction requested=true 2024-11-23T15:40:53,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:53,678 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:53,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e50851c96ec6702e3c34ccb64f6cb9d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:40:53,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:53,678 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:53,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e50851c96ec6702e3c34ccb64f6cb9d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:40:53,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:53,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e50851c96ec6702e3c34ccb64f6cb9d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:40:53,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:53,682 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:53,683 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): e50851c96ec6702e3c34ccb64f6cb9d4/B is initiating minor compaction (all files) 2024-11-23T15:40:53,683 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:53,683 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): e50851c96ec6702e3c34ccb64f6cb9d4/A is initiating minor compaction (all files) 2024-11-23T15:40:53,683 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e50851c96ec6702e3c34ccb64f6cb9d4/B in TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:53,683 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e50851c96ec6702e3c34ccb64f6cb9d4/A in TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:53,683 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/35c532aa098d43fa80fa09c73f52f893, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/d27b474a3ed34f80aa3b584344d9cb23, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/dc226e0965474e108d7f25c4feeee487] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp, totalSize=35.5 K 2024-11-23T15:40:53,683 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/028cb4a96bb54af48231f7e92758b467, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/6c5e18df32aa485bbfd8e8d4ab8b3ac3, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/e62c6e1f745b4d40b191cb775b99f80a] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp, totalSize=107.9 K 2024-11-23T15:40:53,683 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:53,683 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. files: [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/028cb4a96bb54af48231f7e92758b467, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/6c5e18df32aa485bbfd8e8d4ab8b3ac3, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/e62c6e1f745b4d40b191cb775b99f80a] 2024-11-23T15:40:53,683 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 35c532aa098d43fa80fa09c73f52f893, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732376448737 2024-11-23T15:40:53,683 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 028cb4a96bb54af48231f7e92758b467, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732376448737 2024-11-23T15:40:53,684 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting d27b474a3ed34f80aa3b584344d9cb23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732376449916 2024-11-23T15:40:53,685 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c5e18df32aa485bbfd8e8d4ab8b3ac3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732376449916 2024-11-23T15:40:53,685 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting dc226e0965474e108d7f25c4feeee487, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732376451096 2024-11-23T15:40:53,685 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting e62c6e1f745b4d40b191cb775b99f80a, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732376451095 2024-11-23T15:40:53,704 INFO [master/b712f9af2c12:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-23T15:40:53,704 INFO [master/b712f9af2c12:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-23T15:40:53,711 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:53,730 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e50851c96ec6702e3c34ccb64f6cb9d4#B#compaction#148 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:53,731 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/404f2cc02d3a4642b1eed9bd0553b4ef is 50, key is test_row_0/B:col10/1732376451098/Put/seqid=0 2024-11-23T15:40:53,738 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123894c90a7549e4c48a8f7de272b7f0a0d_e50851c96ec6702e3c34ccb64f6cb9d4 store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:53,741 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123894c90a7549e4c48a8f7de272b7f0a0d_e50851c96ec6702e3c34ccb64f6cb9d4, store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:53,741 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123894c90a7549e4c48a8f7de272b7f0a0d_e50851c96ec6702e3c34ccb64f6cb9d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:53,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741996_1172 (size=12459) 2024-11-23T15:40:53,800 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/404f2cc02d3a4642b1eed9bd0553b4ef as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/404f2cc02d3a4642b1eed9bd0553b4ef 2024-11-23T15:40:53,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741997_1173 (size=4469) 2024-11-23T15:40:53,810 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e50851c96ec6702e3c34ccb64f6cb9d4#A#compaction#147 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:53,810 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/f7050ecec00a4ada959e21d51a96c4f4 is 175, key is test_row_0/A:col10/1732376451098/Put/seqid=0 2024-11-23T15:40:53,811 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e50851c96ec6702e3c34ccb64f6cb9d4/B of e50851c96ec6702e3c34ccb64f6cb9d4 into 404f2cc02d3a4642b1eed9bd0553b4ef(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:53,811 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:53,811 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., storeName=e50851c96ec6702e3c34ccb64f6cb9d4/B, priority=13, startTime=1732376453678; duration=0sec 2024-11-23T15:40:53,811 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:53,812 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e50851c96ec6702e3c34ccb64f6cb9d4:B 2024-11-23T15:40:53,812 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:40:53,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-23T15:40:53,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:53,815 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing e50851c96ec6702e3c34ccb64f6cb9d4 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-23T15:40:53,819 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:40:53,820 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): e50851c96ec6702e3c34ccb64f6cb9d4/C is initiating minor compaction (all files) 2024-11-23T15:40:53,820 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e50851c96ec6702e3c34ccb64f6cb9d4/C in TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:53,820 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/8be98f63542a4c1c9032cca088fbe977, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/0fa4cbf7e230457cba0c355d69556208, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/6d14f2df26a54bff8014fc23cf527dff] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp, totalSize=35.5 K 2024-11-23T15:40:53,820 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8be98f63542a4c1c9032cca088fbe977, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732376448737 2024-11-23T15:40:53,821 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fa4cbf7e230457cba0c355d69556208, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732376449916 2024-11-23T15:40:53,821 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d14f2df26a54bff8014fc23cf527dff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732376451096 2024-11-23T15:40:53,850 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e50851c96ec6702e3c34ccb64f6cb9d4#C#compaction#149 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:53,850 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/917844199fe04366827d258e7dc33f59 is 50, key is test_row_0/C:col10/1732376451098/Put/seqid=0 2024-11-23T15:40:53,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741998_1174 (size=31413) 2024-11-23T15:40:53,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c1325903a70e4e8a988cb396008be669_e50851c96ec6702e3c34ccb64f6cb9d4 is 50, key is test_row_0/A:col10/1732376452298/Put/seqid=0 2024-11-23T15:40:53,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T15:40:53,880 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/f7050ecec00a4ada959e21d51a96c4f4 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/f7050ecec00a4ada959e21d51a96c4f4 2024-11-23T15:40:53,888 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e50851c96ec6702e3c34ccb64f6cb9d4/A of e50851c96ec6702e3c34ccb64f6cb9d4 into f7050ecec00a4ada959e21d51a96c4f4(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:53,888 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:53,888 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., storeName=e50851c96ec6702e3c34ccb64f6cb9d4/A, priority=13, startTime=1732376453678; duration=0sec 2024-11-23T15:40:53,888 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:53,888 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e50851c96ec6702e3c34ccb64f6cb9d4:A 2024-11-23T15:40:53,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741999_1175 (size=12459) 2024-11-23T15:40:53,902 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/917844199fe04366827d258e7dc33f59 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/917844199fe04366827d258e7dc33f59 2024-11-23T15:40:53,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742000_1176 (size=12304) 2024-11-23T15:40:53,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:53,911 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e50851c96ec6702e3c34ccb64f6cb9d4/C of e50851c96ec6702e3c34ccb64f6cb9d4 into 917844199fe04366827d258e7dc33f59(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:53,911 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:53,911 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., storeName=e50851c96ec6702e3c34ccb64f6cb9d4/C, priority=13, startTime=1732376453679; duration=0sec 2024-11-23T15:40:53,912 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:53,912 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e50851c96ec6702e3c34ccb64f6cb9d4:C 2024-11-23T15:40:53,919 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c1325903a70e4e8a988cb396008be669_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c1325903a70e4e8a988cb396008be669_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:53,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/8e91bf36582248818b380190eae63eee, store: [table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:53,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/8e91bf36582248818b380190eae63eee is 175, key is test_row_0/A:col10/1732376452298/Put/seqid=0 2024-11-23T15:40:53,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742001_1177 (size=31105) 2024-11-23T15:40:53,948 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/8e91bf36582248818b380190eae63eee 2024-11-23T15:40:53,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/44abd19c53b341b5962346b279906af8 is 50, key is test_row_0/B:col10/1732376452298/Put/seqid=0 2024-11-23T15:40:53,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742002_1178 (size=12151) 2024-11-23T15:40:54,361 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/44abd19c53b341b5962346b279906af8 2024-11-23T15:40:54,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/8cc2c921b5014d9d8ac941ff7fb15de3 is 50, key is test_row_0/C:col10/1732376452298/Put/seqid=0 2024-11-23T15:40:54,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742003_1179 (size=12151) 2024-11-23T15:40:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:54,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:40:54,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376514488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376514489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:54,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376514490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:54,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:54,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376514491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:54,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:54,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376514493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:54,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376514604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:54,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376514604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:54,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376514604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376514604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376514604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:54,815 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/8cc2c921b5014d9d8ac941ff7fb15de3 2024-11-23T15:40:54,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376514815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:54,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376514815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:54,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376514815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:54,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:54,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376514815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:54,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376514815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:54,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:54,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:54,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/8e91bf36582248818b380190eae63eee as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/8e91bf36582248818b380190eae63eee 2024-11-23T15:40:54,827 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/8e91bf36582248818b380190eae63eee, entries=150, sequenceid=155, filesize=30.4 K 2024-11-23T15:40:54,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/44abd19c53b341b5962346b279906af8 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/44abd19c53b341b5962346b279906af8 2024-11-23T15:40:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:54,836 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/44abd19c53b341b5962346b279906af8, entries=150, sequenceid=155, filesize=11.9 K 2024-11-23T15:40:54,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/8cc2c921b5014d9d8ac941ff7fb15de3 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/8cc2c921b5014d9d8ac941ff7fb15de3 2024-11-23T15:40:54,848 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/8cc2c921b5014d9d8ac941ff7fb15de3, entries=150, sequenceid=155, filesize=11.9 K 2024-11-23T15:40:54,850 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e50851c96ec6702e3c34ccb64f6cb9d4 in 1035ms, sequenceid=155, compaction requested=false 2024-11-23T15:40:54,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:54,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:54,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-23T15:40:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-23T15:40:54,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-23T15:40:54,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1140 sec 2024-11-23T15:40:54,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.1200 sec 2024-11-23T15:40:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T15:40:54,884 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:40:54,885 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:40:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-23T15:40:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T15:40:54,886 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:40:54,887 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:40:54,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:40:54,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T15:40:55,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T15:40:55,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:55,039 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing e50851c96ec6702e3c34ccb64f6cb9d4 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-23T15:40:55,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123d656681f401b41e58136f9e422b9913b_e50851c96ec6702e3c34ccb64f6cb9d4 is 50, key is test_row_0/A:col10/1732376454482/Put/seqid=0 2024-11-23T15:40:55,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742004_1180 (size=12304) 2024-11-23T15:40:55,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:55,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:40:55,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376515142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376515146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376515150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376515151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376515151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T15:40:55,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376515254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376515254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376515254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376515254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376515264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:55,457 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123d656681f401b41e58136f9e422b9913b_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123d656681f401b41e58136f9e422b9913b_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:55,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/10e76ea5e84c4943ae8991bef94b978c, store: [table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:55,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/10e76ea5e84c4943ae8991bef94b978c is 175, key is test_row_0/A:col10/1732376454482/Put/seqid=0 2024-11-23T15:40:55,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742005_1181 (size=31105) 2024-11-23T15:40:55,465 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/10e76ea5e84c4943ae8991bef94b978c 2024-11-23T15:40:55,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376515464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376515465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376515465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376515465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/0705e430eddf42a2b456e221b241bda6 is 50, key is test_row_0/B:col10/1732376454482/Put/seqid=0 2024-11-23T15:40:55,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376515475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742006_1182 (size=12151) 2024-11-23T15:40:55,481 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/0705e430eddf42a2b456e221b241bda6 2024-11-23T15:40:55,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/a5c56ac811654afb9e83c3d556d51e75 is 50, key is test_row_0/C:col10/1732376454482/Put/seqid=0 2024-11-23T15:40:55,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742007_1183 (size=12151) 2024-11-23T15:40:55,501 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/a5c56ac811654afb9e83c3d556d51e75 2024-11-23T15:40:55,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/10e76ea5e84c4943ae8991bef94b978c as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/10e76ea5e84c4943ae8991bef94b978c 2024-11-23T15:40:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T15:40:55,515 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/10e76ea5e84c4943ae8991bef94b978c, entries=150, sequenceid=175, filesize=30.4 K 2024-11-23T15:40:55,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/0705e430eddf42a2b456e221b241bda6 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/0705e430eddf42a2b456e221b241bda6 2024-11-23T15:40:55,522 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/0705e430eddf42a2b456e221b241bda6, entries=150, sequenceid=175, filesize=11.9 K 2024-11-23T15:40:55,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/a5c56ac811654afb9e83c3d556d51e75 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/a5c56ac811654afb9e83c3d556d51e75 2024-11-23T15:40:55,531 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/a5c56ac811654afb9e83c3d556d51e75, entries=150, sequenceid=175, filesize=11.9 K 2024-11-23T15:40:55,532 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for e50851c96ec6702e3c34ccb64f6cb9d4 in 493ms, sequenceid=175, compaction requested=true 2024-11-23T15:40:55,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:55,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:55,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-23T15:40:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-23T15:40:55,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-23T15:40:55,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 647 msec 2024-11-23T15:40:55,539 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 653 msec 2024-11-23T15:40:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:55,777 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:40:55,777 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e50851c96ec6702e3c34ccb64f6cb9d4 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-23T15:40:55,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376515792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232dbdb353c89b4caeacec356ab4820dab_e50851c96ec6702e3c34ccb64f6cb9d4 is 50, key is test_row_0/A:col10/1732376455149/Put/seqid=0 2024-11-23T15:40:55,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376515799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376515799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376515802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376515801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742008_1184 (size=14794) 2024-11-23T15:40:55,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376515915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376515915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376515915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376515915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376515915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:55,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:55,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:56,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T15:40:56,024 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:40:56,024 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:40:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-23T15:40:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T15:40:56,026 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:40:56,027 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:40:56,027 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:40:56,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376516124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:56,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376516124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:56,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376516124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:56,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:56,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376516124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:56,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:56,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:56,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376516124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T15:40:56,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T15:40:56,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:56,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:40:56,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:56,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:56,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:56,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:56,222 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:56,226 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232dbdb353c89b4caeacec356ab4820dab_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232dbdb353c89b4caeacec356ab4820dab_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:56,227 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/5c61a949652648aeb32f1297b0acef40, store: [table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:56,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/5c61a949652648aeb32f1297b0acef40 is 175, key is test_row_0/A:col10/1732376455149/Put/seqid=0 2024-11-23T15:40:56,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742009_1185 (size=39749) 2024-11-23T15:40:56,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T15:40:56,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:56,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:40:56,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:56,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:56,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:56,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:56,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T15:40:56,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376516434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:56,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376516434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:56,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:56,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:56,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376516434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:56,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:56,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376516434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:56,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376516444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:56,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T15:40:56,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:56,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:40:56,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:56,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:56,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:56,632 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/5c61a949652648aeb32f1297b0acef40 2024-11-23T15:40:56,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T15:40:56,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:56,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:40:56,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:56,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:56,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:56,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/65744128ecaf420fb7fc36b28bcddebb is 50, key is test_row_0/B:col10/1732376455149/Put/seqid=0 2024-11-23T15:40:56,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742010_1186 (size=12151) 2024-11-23T15:40:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T15:40:56,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T15:40:56,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:56,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:40:56,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:56,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:56,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:56,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T15:40:56,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:56,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:40:56,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:56,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:56,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:56,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376516944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:56,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:56,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376516944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376516944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376516945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:56,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:56,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376516954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:56,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:56,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:57,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/65744128ecaf420fb7fc36b28bcddebb 2024-11-23T15:40:57,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/92d78ad0743442eab084cd0f9e12e07a is 50, key is test_row_0/C:col10/1732376455149/Put/seqid=0 2024-11-23T15:40:57,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742011_1187 (size=12151) 2024-11-23T15:40:57,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T15:40:57,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:57,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:40:57,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:57,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:57,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T15:40:57,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T15:40:57,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:57,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:40:57,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:57,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:57,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:57,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T15:40:57,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:57,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:40:57,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:57,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:57,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:40:57,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/92d78ad0743442eab084cd0f9e12e07a 2024-11-23T15:40:57,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/5c61a949652648aeb32f1297b0acef40 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/5c61a949652648aeb32f1297b0acef40 2024-11-23T15:40:57,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/5c61a949652648aeb32f1297b0acef40, entries=200, sequenceid=195, filesize=38.8 K 2024-11-23T15:40:57,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/65744128ecaf420fb7fc36b28bcddebb as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/65744128ecaf420fb7fc36b28bcddebb 2024-11-23T15:40:57,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/65744128ecaf420fb7fc36b28bcddebb, entries=150, sequenceid=195, filesize=11.9 K 2024-11-23T15:40:57,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/92d78ad0743442eab084cd0f9e12e07a as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/92d78ad0743442eab084cd0f9e12e07a 2024-11-23T15:40:57,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/92d78ad0743442eab084cd0f9e12e07a, entries=150, sequenceid=195, filesize=11.9 K 2024-11-23T15:40:57,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for e50851c96ec6702e3c34ccb64f6cb9d4 in 1704ms, sequenceid=195, compaction requested=true 2024-11-23T15:40:57,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:57,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e50851c96ec6702e3c34ccb64f6cb9d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:40:57,481 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:40:57,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:57,481 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:40:57,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e50851c96ec6702e3c34ccb64f6cb9d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:40:57,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:57,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e50851c96ec6702e3c34ccb64f6cb9d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:40:57,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:57,482 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:40:57,482 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133372 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:40:57,483 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): e50851c96ec6702e3c34ccb64f6cb9d4/B is initiating minor compaction (all files) 2024-11-23T15:40:57,483 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): e50851c96ec6702e3c34ccb64f6cb9d4/A is initiating minor compaction (all files) 2024-11-23T15:40:57,483 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e50851c96ec6702e3c34ccb64f6cb9d4/B in TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:57,483 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e50851c96ec6702e3c34ccb64f6cb9d4/A in TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:57,483 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/404f2cc02d3a4642b1eed9bd0553b4ef, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/44abd19c53b341b5962346b279906af8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/0705e430eddf42a2b456e221b241bda6, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/65744128ecaf420fb7fc36b28bcddebb] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp, totalSize=47.8 K 2024-11-23T15:40:57,483 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/f7050ecec00a4ada959e21d51a96c4f4, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/8e91bf36582248818b380190eae63eee, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/10e76ea5e84c4943ae8991bef94b978c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/5c61a949652648aeb32f1297b0acef40] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp, totalSize=130.2 K 2024-11-23T15:40:57,483 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:57,483 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. files: [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/f7050ecec00a4ada959e21d51a96c4f4, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/8e91bf36582248818b380190eae63eee, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/10e76ea5e84c4943ae8991bef94b978c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/5c61a949652648aeb32f1297b0acef40] 2024-11-23T15:40:57,483 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 404f2cc02d3a4642b1eed9bd0553b4ef, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732376451096 2024-11-23T15:40:57,483 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7050ecec00a4ada959e21d51a96c4f4, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732376451096 2024-11-23T15:40:57,483 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e91bf36582248818b380190eae63eee, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732376452285 2024-11-23T15:40:57,483 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 44abd19c53b341b5962346b279906af8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732376452285 2024-11-23T15:40:57,484 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10e76ea5e84c4943ae8991bef94b978c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732376454482 2024-11-23T15:40:57,484 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 0705e430eddf42a2b456e221b241bda6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732376454482 2024-11-23T15:40:57,484 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c61a949652648aeb32f1297b0acef40, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732376455140 2024-11-23T15:40:57,484 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 65744128ecaf420fb7fc36b28bcddebb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732376455140 2024-11-23T15:40:57,491 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:57,492 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e50851c96ec6702e3c34ccb64f6cb9d4#B#compaction#159 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:57,493 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/d8a305e267164f8c9084162174ca2658 is 50, key is test_row_0/B:col10/1732376455149/Put/seqid=0 2024-11-23T15:40:57,493 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411239c79fbfaf32147f1b1ae4eb9ef4b8019_e50851c96ec6702e3c34ccb64f6cb9d4 store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:57,496 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411239c79fbfaf32147f1b1ae4eb9ef4b8019_e50851c96ec6702e3c34ccb64f6cb9d4, store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:57,496 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411239c79fbfaf32147f1b1ae4eb9ef4b8019_e50851c96ec6702e3c34ccb64f6cb9d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:57,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742012_1188 (size=12595) 2024-11-23T15:40:57,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742013_1189 (size=4469) 2024-11-23T15:40:57,509 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e50851c96ec6702e3c34ccb64f6cb9d4#A#compaction#160 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:57,509 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/1a37d920bd034953a37a9c319ef06e4d is 175, key is test_row_0/A:col10/1732376455149/Put/seqid=0 2024-11-23T15:40:57,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742014_1190 (size=31549) 2024-11-23T15:40:57,518 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/1a37d920bd034953a37a9c319ef06e4d as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/1a37d920bd034953a37a9c319ef06e4d 2024-11-23T15:40:57,525 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e50851c96ec6702e3c34ccb64f6cb9d4/A of e50851c96ec6702e3c34ccb64f6cb9d4 into 1a37d920bd034953a37a9c319ef06e4d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:57,525 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:57,525 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., storeName=e50851c96ec6702e3c34ccb64f6cb9d4/A, priority=12, startTime=1732376457481; duration=0sec 2024-11-23T15:40:57,525 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:40:57,525 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e50851c96ec6702e3c34ccb64f6cb9d4:A 2024-11-23T15:40:57,525 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:40:57,527 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:40:57,527 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): e50851c96ec6702e3c34ccb64f6cb9d4/C is initiating minor compaction (all files) 2024-11-23T15:40:57,527 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e50851c96ec6702e3c34ccb64f6cb9d4/C in TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:57,527 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/917844199fe04366827d258e7dc33f59, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/8cc2c921b5014d9d8ac941ff7fb15de3, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/a5c56ac811654afb9e83c3d556d51e75, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/92d78ad0743442eab084cd0f9e12e07a] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp, totalSize=47.8 K 2024-11-23T15:40:57,528 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 917844199fe04366827d258e7dc33f59, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732376451096 2024-11-23T15:40:57,528 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cc2c921b5014d9d8ac941ff7fb15de3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732376452285 2024-11-23T15:40:57,529 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5c56ac811654afb9e83c3d556d51e75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732376454482 2024-11-23T15:40:57,529 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92d78ad0743442eab084cd0f9e12e07a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732376455140 2024-11-23T15:40:57,543 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e50851c96ec6702e3c34ccb64f6cb9d4#C#compaction#161 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:40:57,544 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/f99dd901d53c409a8e06aa9b04c530ad is 50, key is test_row_0/C:col10/1732376455149/Put/seqid=0 2024-11-23T15:40:57,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T15:40:57,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742015_1191 (size=12595) 2024-11-23T15:40:57,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:57,556 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing e50851c96ec6702e3c34ccb64f6cb9d4 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-23T15:40:57,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123596eef4ab3b8494da5757aa481f1fbdf_e50851c96ec6702e3c34ccb64f6cb9d4 is 50, key is test_row_0/A:col10/1732376455797/Put/seqid=0 2024-11-23T15:40:57,566 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/f99dd901d53c409a8e06aa9b04c530ad as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/f99dd901d53c409a8e06aa9b04c530ad 2024-11-23T15:40:57,573 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e50851c96ec6702e3c34ccb64f6cb9d4/C of e50851c96ec6702e3c34ccb64f6cb9d4 into f99dd901d53c409a8e06aa9b04c530ad(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:57,574 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:57,574 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., storeName=e50851c96ec6702e3c34ccb64f6cb9d4/C, priority=12, startTime=1732376457481; duration=0sec 2024-11-23T15:40:57,574 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:57,574 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e50851c96ec6702e3c34ccb64f6cb9d4:C 2024-11-23T15:40:57,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742016_1192 (size=12304) 2024-11-23T15:40:57,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:57,583 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123596eef4ab3b8494da5757aa481f1fbdf_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123596eef4ab3b8494da5757aa481f1fbdf_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:57,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/a5b93532a4f2414cad04b62f86e8ddab, store: [table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:57,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/a5b93532a4f2414cad04b62f86e8ddab is 175, key is test_row_0/A:col10/1732376455797/Put/seqid=0 2024-11-23T15:40:57,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742017_1193 (size=31105) 2024-11-23T15:40:57,592 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/a5b93532a4f2414cad04b62f86e8ddab 2024-11-23T15:40:57,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/4662961ff57e41bb9228eae41c84aad0 is 50, key is test_row_0/B:col10/1732376455797/Put/seqid=0 2024-11-23T15:40:57,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742018_1194 (size=12151) 2024-11-23T15:40:57,903 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/d8a305e267164f8c9084162174ca2658 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/d8a305e267164f8c9084162174ca2658 2024-11-23T15:40:57,912 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e50851c96ec6702e3c34ccb64f6cb9d4/B of e50851c96ec6702e3c34ccb64f6cb9d4 into d8a305e267164f8c9084162174ca2658(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:40:57,912 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:57,912 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., storeName=e50851c96ec6702e3c34ccb64f6cb9d4/B, priority=12, startTime=1732376457481; duration=0sec 2024-11-23T15:40:57,912 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:40:57,912 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e50851c96ec6702e3c34ccb64f6cb9d4:B 2024-11-23T15:40:57,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:57,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:40:58,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376517976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376517976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,013 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/4662961ff57e41bb9228eae41c84aad0 2024-11-23T15:40:58,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376518008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376518008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376518008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/f1c40746ef4c4b62917b141285bd1949 is 50, key is test_row_0/C:col10/1732376455797/Put/seqid=0 2024-11-23T15:40:58,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742019_1195 (size=12151) 2024-11-23T15:40:58,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376518114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376518114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376518124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376518124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376518124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T15:40:58,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376518324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376518324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376518334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376518334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376518334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,434 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/f1c40746ef4c4b62917b141285bd1949 2024-11-23T15:40:58,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/a5b93532a4f2414cad04b62f86e8ddab as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/a5b93532a4f2414cad04b62f86e8ddab 2024-11-23T15:40:58,444 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/a5b93532a4f2414cad04b62f86e8ddab, entries=150, sequenceid=212, filesize=30.4 K 2024-11-23T15:40:58,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/4662961ff57e41bb9228eae41c84aad0 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/4662961ff57e41bb9228eae41c84aad0 2024-11-23T15:40:58,449 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/4662961ff57e41bb9228eae41c84aad0, entries=150, sequenceid=212, filesize=11.9 K 2024-11-23T15:40:58,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/f1c40746ef4c4b62917b141285bd1949 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/f1c40746ef4c4b62917b141285bd1949 2024-11-23T15:40:58,457 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/f1c40746ef4c4b62917b141285bd1949, entries=150, sequenceid=212, filesize=11.9 K 2024-11-23T15:40:58,458 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for e50851c96ec6702e3c34ccb64f6cb9d4 in 902ms, sequenceid=212, compaction requested=false 2024-11-23T15:40:58,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:40:58,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:40:58,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-23T15:40:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-23T15:40:58,463 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-23T15:40:58,463 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4340 sec 2024-11-23T15:40:58,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.4390 sec 2024-11-23T15:40:58,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:58,639 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:40:58,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e50851c96ec6702e3c34ccb64f6cb9d4 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-23T15:40:58,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c9bf39a5c57c40d69564ed132d9948ea_e50851c96ec6702e3c34ccb64f6cb9d4 is 50, key is test_row_0/A:col10/1732376457974/Put/seqid=0 2024-11-23T15:40:58,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376518651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742020_1196 (size=17284) 2024-11-23T15:40:58,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376518654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376518655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376518655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376518656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376518764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376518764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376518764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376518774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376518775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376518974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376518974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376518974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376518984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,984 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:58,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:58,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376518984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:58,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:58,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:59,060 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:40:59,064 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c9bf39a5c57c40d69564ed132d9948ea_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c9bf39a5c57c40d69564ed132d9948ea_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:40:59,065 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/d63f373292ca489b8ba9c646b7aa280c, store: [table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:40:59,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/d63f373292ca489b8ba9c646b7aa280c is 175, key is test_row_0/A:col10/1732376457974/Put/seqid=0 2024-11-23T15:40:59,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742021_1197 (size=48389) 2024-11-23T15:40:59,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:59,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376519284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:59,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:59,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376519284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:59,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:59,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:59,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376519284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:59,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:59,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:59,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376519294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:59,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:59,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:59,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:59,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:59,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376519294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:59,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:59,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:59,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:59,470 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/d63f373292ca489b8ba9c646b7aa280c 2024-11-23T15:40:59,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/f46f888bedbc43758f7cbe55d77dd328 is 50, key is test_row_0/B:col10/1732376457974/Put/seqid=0 2024-11-23T15:40:59,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742022_1198 (size=12151) 2024-11-23T15:40:59,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:59,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376519794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:59,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:59,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:59,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376519794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:59,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:59,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:59,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:59,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376519794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:59,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:59,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:59,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:59,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:59,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:59,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:59,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376519804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:59,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:40:59,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:59,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376519804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:40:59,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:59,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:59,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:59,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:40:59,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:40:59,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/f46f888bedbc43758f7cbe55d77dd328 2024-11-23T15:40:59,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/1ce21fb735024a7eac7f9b608331bbc6 is 50, key is test_row_0/C:col10/1732376457974/Put/seqid=0 2024-11-23T15:40:59,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742023_1199 (size=12151) 2024-11-23T15:41:00,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T15:41:00,208 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:00,208 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:41:00,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-23T15:41:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-23T15:41:00,210 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:41:00,211 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:41:00,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:41:00,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/1ce21fb735024a7eac7f9b608331bbc6 2024-11-23T15:41:00,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/d63f373292ca489b8ba9c646b7aa280c as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/d63f373292ca489b8ba9c646b7aa280c 2024-11-23T15:41:00,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/d63f373292ca489b8ba9c646b7aa280c, entries=250, sequenceid=237, filesize=47.3 K 2024-11-23T15:41:00,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/f46f888bedbc43758f7cbe55d77dd328 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/f46f888bedbc43758f7cbe55d77dd328 2024-11-23T15:41:00,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/f46f888bedbc43758f7cbe55d77dd328, entries=150, sequenceid=237, filesize=11.9 K 2024-11-23T15:41:00,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/1ce21fb735024a7eac7f9b608331bbc6 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/1ce21fb735024a7eac7f9b608331bbc6 2024-11-23T15:41:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-23T15:41:00,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/1ce21fb735024a7eac7f9b608331bbc6, entries=150, sequenceid=237, filesize=11.9 K 2024-11-23T15:41:00,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for e50851c96ec6702e3c34ccb64f6cb9d4 in 1680ms, sequenceid=237, compaction requested=true 2024-11-23T15:41:00,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:41:00,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e50851c96ec6702e3c34ccb64f6cb9d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:41:00,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:00,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e50851c96ec6702e3c34ccb64f6cb9d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:41:00,319 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:00,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:00,319 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:00,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e50851c96ec6702e3c34ccb64f6cb9d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:41:00,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:00,320 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:00,320 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111043 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:00,320 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): e50851c96ec6702e3c34ccb64f6cb9d4/B is initiating minor compaction (all files) 2024-11-23T15:41:00,320 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): e50851c96ec6702e3c34ccb64f6cb9d4/A is initiating minor compaction (all files) 2024-11-23T15:41:00,321 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e50851c96ec6702e3c34ccb64f6cb9d4/B in TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:00,321 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e50851c96ec6702e3c34ccb64f6cb9d4/A in TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:00,321 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/d8a305e267164f8c9084162174ca2658, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/4662961ff57e41bb9228eae41c84aad0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/f46f888bedbc43758f7cbe55d77dd328] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp, totalSize=36.0 K 2024-11-23T15:41:00,321 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/1a37d920bd034953a37a9c319ef06e4d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/a5b93532a4f2414cad04b62f86e8ddab, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/d63f373292ca489b8ba9c646b7aa280c] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp, totalSize=108.4 K 2024-11-23T15:41:00,321 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:00,321 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. files: [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/1a37d920bd034953a37a9c319ef06e4d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/a5b93532a4f2414cad04b62f86e8ddab, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/d63f373292ca489b8ba9c646b7aa280c] 2024-11-23T15:41:00,321 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a37d920bd034953a37a9c319ef06e4d, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732376455140 2024-11-23T15:41:00,321 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting d8a305e267164f8c9084162174ca2658, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732376455140 2024-11-23T15:41:00,321 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5b93532a4f2414cad04b62f86e8ddab, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732376455797 2024-11-23T15:41:00,321 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 4662961ff57e41bb9228eae41c84aad0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732376455797 2024-11-23T15:41:00,322 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting d63f373292ca489b8ba9c646b7aa280c, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732376457974 2024-11-23T15:41:00,322 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting f46f888bedbc43758f7cbe55d77dd328, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732376457974 2024-11-23T15:41:00,328 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:41:00,329 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e50851c96ec6702e3c34ccb64f6cb9d4#B#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:00,330 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/82d17813839047998f0d8df1e68778b8 is 50, key is test_row_0/B:col10/1732376457974/Put/seqid=0 2024-11-23T15:41:00,333 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411237b93b831ef684fe48775b31ad8c76b23_e50851c96ec6702e3c34ccb64f6cb9d4 store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:41:00,335 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411237b93b831ef684fe48775b31ad8c76b23_e50851c96ec6702e3c34ccb64f6cb9d4, store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:41:00,336 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411237b93b831ef684fe48775b31ad8c76b23_e50851c96ec6702e3c34ccb64f6cb9d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:41:00,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742024_1200 (size=12697) 2024-11-23T15:41:00,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742025_1201 (size=4469) 2024-11-23T15:41:00,345 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e50851c96ec6702e3c34ccb64f6cb9d4#A#compaction#169 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:00,347 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/0e9f6d11debb404ba5a3a29e85e23a34 is 175, key is test_row_0/A:col10/1732376457974/Put/seqid=0 2024-11-23T15:41:00,349 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/82d17813839047998f0d8df1e68778b8 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/82d17813839047998f0d8df1e68778b8 2024-11-23T15:41:00,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742026_1202 (size=31651) 2024-11-23T15:41:00,356 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e50851c96ec6702e3c34ccb64f6cb9d4/B of e50851c96ec6702e3c34ccb64f6cb9d4 into 82d17813839047998f0d8df1e68778b8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:00,356 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:41:00,357 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., storeName=e50851c96ec6702e3c34ccb64f6cb9d4/B, priority=13, startTime=1732376460319; duration=0sec 2024-11-23T15:41:00,357 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:00,357 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e50851c96ec6702e3c34ccb64f6cb9d4:B 2024-11-23T15:41:00,357 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:00,358 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:00,358 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): e50851c96ec6702e3c34ccb64f6cb9d4/C is initiating minor compaction (all files) 2024-11-23T15:41:00,358 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e50851c96ec6702e3c34ccb64f6cb9d4/C in TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:00,358 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/f99dd901d53c409a8e06aa9b04c530ad, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/f1c40746ef4c4b62917b141285bd1949, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/1ce21fb735024a7eac7f9b608331bbc6] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp, totalSize=36.0 K 2024-11-23T15:41:00,359 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting f99dd901d53c409a8e06aa9b04c530ad, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732376455140 2024-11-23T15:41:00,359 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting f1c40746ef4c4b62917b141285bd1949, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732376455797 2024-11-23T15:41:00,360 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ce21fb735024a7eac7f9b608331bbc6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732376457974 2024-11-23T15:41:00,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-23T15:41:00,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:00,363 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing e50851c96ec6702e3c34ccb64f6cb9d4 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T15:41:00,368 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e50851c96ec6702e3c34ccb64f6cb9d4#C#compaction#170 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:00,368 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/d09001c5f4a046c28fdab73950de70fb is 50, key is test_row_0/C:col10/1732376457974/Put/seqid=0 2024-11-23T15:41:00,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123b1e0ad1a6ee544ed91f0cf88d9cb6077_e50851c96ec6702e3c34ccb64f6cb9d4 is 50, key is test_row_0/A:col10/1732376458654/Put/seqid=0 2024-11-23T15:41:00,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742027_1203 (size=12697) 2024-11-23T15:41:00,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742028_1204 (size=12304) 2024-11-23T15:41:00,389 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/d09001c5f4a046c28fdab73950de70fb as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/d09001c5f4a046c28fdab73950de70fb 2024-11-23T15:41:00,394 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e50851c96ec6702e3c34ccb64f6cb9d4/C of e50851c96ec6702e3c34ccb64f6cb9d4 into d09001c5f4a046c28fdab73950de70fb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:00,394 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:41:00,394 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., storeName=e50851c96ec6702e3c34ccb64f6cb9d4/C, priority=13, startTime=1732376460319; duration=0sec 2024-11-23T15:41:00,394 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:00,395 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e50851c96ec6702e3c34ccb64f6cb9d4:C 2024-11-23T15:41:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-23T15:41:00,758 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/0e9f6d11debb404ba5a3a29e85e23a34 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/0e9f6d11debb404ba5a3a29e85e23a34 2024-11-23T15:41:00,763 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e50851c96ec6702e3c34ccb64f6cb9d4/A of e50851c96ec6702e3c34ccb64f6cb9d4 into 0e9f6d11debb404ba5a3a29e85e23a34(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:00,763 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:41:00,763 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., storeName=e50851c96ec6702e3c34ccb64f6cb9d4/A, priority=13, startTime=1732376460319; duration=0sec 2024-11-23T15:41:00,763 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:00,763 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e50851c96ec6702e3c34ccb64f6cb9d4:A 2024-11-23T15:41:00,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:00,790 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123b1e0ad1a6ee544ed91f0cf88d9cb6077_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b1e0ad1a6ee544ed91f0cf88d9cb6077_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:00,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/8469c0440bac429da17af741093c2309, store: [table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:41:00,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/8469c0440bac429da17af741093c2309 is 175, key is test_row_0/A:col10/1732376458654/Put/seqid=0 2024-11-23T15:41:00,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742029_1205 (size=31105) 2024-11-23T15:41:00,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:00,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:41:00,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-23T15:41:00,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:00,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376520862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:00,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:00,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:00,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:00,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:00,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376520863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:00,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:00,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:00,864 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:00,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:00,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376520863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:00,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:00,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:00,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:00,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:00,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376520863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:00,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:00,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:00,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:00,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:00,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376520864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:00,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:00,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:00,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:00,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376520974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:00,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:00,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376520974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:00,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:00,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:00,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:00,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376520974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:00,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:00,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:00,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:00,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:00,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376520974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:00,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:00,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:00,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:00,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:00,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:00,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376520974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:00,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:00,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:00,978 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:00,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:00,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:00,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:01,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376521184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:01,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:01,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376521184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:01,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:01,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376521184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376521184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:01,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376521184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:01,196 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/8469c0440bac429da17af741093c2309 2024-11-23T15:41:01,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/6719e502ade244f08bcface8bc4e4ea8 is 50, key is test_row_0/B:col10/1732376458654/Put/seqid=0 2024-11-23T15:41:01,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742030_1206 (size=12151) 2024-11-23T15:41:01,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-23T15:41:01,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376521494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:01,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:01,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:01,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376521494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:01,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:01,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:01,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:01,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376521494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:01,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:01,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376521494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:01,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376521494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:01,608 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/6719e502ade244f08bcface8bc4e4ea8 2024-11-23T15:41:01,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/686177469b5142eda682c96d4feeb327 is 50, key is test_row_0/C:col10/1732376458654/Put/seqid=0 2024-11-23T15:41:01,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742031_1207 (size=12151) 2024-11-23T15:41:02,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376522004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:02,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:02,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376522004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:02,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:02,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:02,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376522004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:02,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376522004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:02,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376522005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:02,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:02,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:02,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:02,020 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/686177469b5142eda682c96d4feeb327 2024-11-23T15:41:02,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/8469c0440bac429da17af741093c2309 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/8469c0440bac429da17af741093c2309 2024-11-23T15:41:02,030 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/8469c0440bac429da17af741093c2309, entries=150, sequenceid=252, filesize=30.4 K 2024-11-23T15:41:02,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/6719e502ade244f08bcface8bc4e4ea8 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/6719e502ade244f08bcface8bc4e4ea8 2024-11-23T15:41:02,035 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/6719e502ade244f08bcface8bc4e4ea8, entries=150, sequenceid=252, filesize=11.9 K 2024-11-23T15:41:02,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/686177469b5142eda682c96d4feeb327 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/686177469b5142eda682c96d4feeb327 2024-11-23T15:41:02,040 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/686177469b5142eda682c96d4feeb327, entries=150, sequenceid=252, filesize=11.9 K 2024-11-23T15:41:02,041 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for e50851c96ec6702e3c34ccb64f6cb9d4 in 1678ms, sequenceid=252, compaction requested=false 2024-11-23T15:41:02,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:41:02,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:02,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-23T15:41:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-23T15:41:02,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-23T15:41:02,044 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8320 sec 2024-11-23T15:41:02,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.8370 sec 2024-11-23T15:41:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-23T15:41:02,355 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:02,355 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:41:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-23T15:41:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-23T15:41:02,357 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:41:02,357 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:41:02,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:41:02,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-23T15:41:02,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-23T15:41:02,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:02,510 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing e50851c96ec6702e3c34ccb64f6cb9d4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T15:41:02,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123fdb8ee3835d44e1295af36389ffceddd_e50851c96ec6702e3c34ccb64f6cb9d4 is 50, key is test_row_0/A:col10/1732376460862/Put/seqid=0 2024-11-23T15:41:02,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742032_1208 (size=12454) 2024-11-23T15:41:02,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,528 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123fdb8ee3835d44e1295af36389ffceddd_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123fdb8ee3835d44e1295af36389ffceddd_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:02,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/ba2380618c9c4d048eea84238c0943e8, store: [table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:41:02,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/ba2380618c9c4d048eea84238c0943e8 is 175, key is test_row_0/A:col10/1732376460862/Put/seqid=0 2024-11-23T15:41:02,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742033_1209 (size=31255) 2024-11-23T15:41:02,537 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/ba2380618c9c4d048eea84238c0943e8 2024-11-23T15:41:02,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/4f199fbdd48b4bd4b0180abf5ddfd167 is 50, key is test_row_0/B:col10/1732376460862/Put/seqid=0 2024-11-23T15:41:02,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742034_1210 (size=12301) 2024-11-23T15:41:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-23T15:41:02,949 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/4f199fbdd48b4bd4b0180abf5ddfd167 2024-11-23T15:41:02,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/e496ff4a643d47b08aeaeeafae97e2e7 is 50, key is test_row_0/C:col10/1732376460862/Put/seqid=0 2024-11-23T15:41:02,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742035_1211 (size=12301) 2024-11-23T15:41:02,965 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/e496ff4a643d47b08aeaeeafae97e2e7 2024-11-23T15:41:02,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/ba2380618c9c4d048eea84238c0943e8 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ba2380618c9c4d048eea84238c0943e8 2024-11-23T15:41:02,975 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ba2380618c9c4d048eea84238c0943e8, entries=150, sequenceid=277, filesize=30.5 K 2024-11-23T15:41:02,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/4f199fbdd48b4bd4b0180abf5ddfd167 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/4f199fbdd48b4bd4b0180abf5ddfd167 2024-11-23T15:41:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,981 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/4f199fbdd48b4bd4b0180abf5ddfd167, entries=150, sequenceid=277, filesize=12.0 K 2024-11-23T15:41:02,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/e496ff4a643d47b08aeaeeafae97e2e7 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/e496ff4a643d47b08aeaeeafae97e2e7 2024-11-23T15:41:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-23T15:41:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,987 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/e496ff4a643d47b08aeaeeafae97e2e7, entries=150, sequenceid=277, filesize=12.0 K 2024-11-23T15:41:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,989 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for e50851c96ec6702e3c34ccb64f6cb9d4 in 479ms, sequenceid=277, compaction requested=true 2024-11-23T15:41:02,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:41:02,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:02,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-23T15:41:02,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-23T15:41:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,993 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-23T15:41:02,993 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 633 msec 2024-11-23T15:41:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,997 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 639 msec 2024-11-23T15:41:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:02,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:03,071 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:41:03,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e50851c96ec6702e3c34ccb64f6cb9d4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T15:41:03,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112327b2f3b1edec412396308e98b0ee3081_e50851c96ec6702e3c34ccb64f6cb9d4 is 50, key is test_row_0/A:col10/1732376463070/Put/seqid=0 2024-11-23T15:41:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742036_1212 (size=20074) 2024-11-23T15:41:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376523119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:03,125 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:03,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376523121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:03,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:03,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376523122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:03,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:03,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:03,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376523122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:03,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:03,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376523126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:03,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:03,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:03,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376523234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:03,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376523234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:03,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376523235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:03,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376523235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:03,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:03,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376523244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:03,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376523444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:03,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376523444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:03,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376523445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:03,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376523445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:03,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376523454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:03,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-23T15:41:03,494 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:03,495 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:41:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-23T15:41:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-23T15:41:03,496 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:41:03,497 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:41:03,497 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:41:03,504 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:03,508 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112327b2f3b1edec412396308e98b0ee3081_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112327b2f3b1edec412396308e98b0ee3081_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:03,509 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/ae7a2bb18eec4328ac7cb7499cfe4263, store: [table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:41:03,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/ae7a2bb18eec4328ac7cb7499cfe4263 is 175, key is test_row_0/A:col10/1732376463070/Put/seqid=0 2024-11-23T15:41:03,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742037_1213 (size=57333) 2024-11-23T15:41:03,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-23T15:41:03,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-23T15:41:03,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:03,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:41:03,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:03,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:03,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:03,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376523754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:03,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376523754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:03,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:03,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376523764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:03,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376523764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:03,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376523764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:03,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-23T15:41:03,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:03,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:41:03,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:03,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:03,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:03,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:03,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-23T15:41:03,914 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/ae7a2bb18eec4328ac7cb7499cfe4263 2024-11-23T15:41:03,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/ee188b5f17114571afbd555071682111 is 50, key is test_row_0/B:col10/1732376463070/Put/seqid=0 2024-11-23T15:41:03,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742038_1214 (size=12301) 2024-11-23T15:41:03,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-23T15:41:03,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:03,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:41:03,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:03,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:03,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:03,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:04,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-23T15:41:04,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:04,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:41:04,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:04,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:04,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:04,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:04,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-23T15:41:04,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-23T15:41:04,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:04,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:41:04,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:04,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:04,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:04,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:04,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376524264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:04,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:04,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:04,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:04,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376524264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:04,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:04,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:04,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:04,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376524274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:04,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:04,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:04,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376524274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:04,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:04,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:04,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:04,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:04,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376524275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:04,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:04,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:04,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:04,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/ee188b5f17114571afbd555071682111 2024-11-23T15:41:04,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/cc11434381d44e57bf94818462a0de5c is 50, key is test_row_0/C:col10/1732376463070/Put/seqid=0 2024-11-23T15:41:04,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742039_1215 (size=12301) 2024-11-23T15:41:04,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/cc11434381d44e57bf94818462a0de5c 2024-11-23T15:41:04,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/ae7a2bb18eec4328ac7cb7499cfe4263 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ae7a2bb18eec4328ac7cb7499cfe4263 2024-11-23T15:41:04,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ae7a2bb18eec4328ac7cb7499cfe4263, entries=300, sequenceid=288, filesize=56.0 K 2024-11-23T15:41:04,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/ee188b5f17114571afbd555071682111 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/ee188b5f17114571afbd555071682111 2024-11-23T15:41:04,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/ee188b5f17114571afbd555071682111, entries=150, sequenceid=288, filesize=12.0 K 2024-11-23T15:41:04,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/cc11434381d44e57bf94818462a0de5c as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/cc11434381d44e57bf94818462a0de5c 2024-11-23T15:41:04,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/cc11434381d44e57bf94818462a0de5c, entries=150, sequenceid=288, filesize=12.0 K 2024-11-23T15:41:04,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e50851c96ec6702e3c34ccb64f6cb9d4 in 1286ms, sequenceid=288, compaction requested=true 2024-11-23T15:41:04,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:41:04,358 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:41:04,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e50851c96ec6702e3c34ccb64f6cb9d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:41:04,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:04,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e50851c96ec6702e3c34ccb64f6cb9d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:41:04,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:04,358 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:41:04,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e50851c96ec6702e3c34ccb64f6cb9d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:41:04,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:04,359 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:41:04,359 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): e50851c96ec6702e3c34ccb64f6cb9d4/A is initiating minor compaction (all files) 2024-11-23T15:41:04,360 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e50851c96ec6702e3c34ccb64f6cb9d4/A in TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:04,360 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/0e9f6d11debb404ba5a3a29e85e23a34, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/8469c0440bac429da17af741093c2309, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ba2380618c9c4d048eea84238c0943e8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ae7a2bb18eec4328ac7cb7499cfe4263] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp, totalSize=147.8 K 2024-11-23T15:41:04,360 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:04,360 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. files: [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/0e9f6d11debb404ba5a3a29e85e23a34, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/8469c0440bac429da17af741093c2309, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ba2380618c9c4d048eea84238c0943e8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ae7a2bb18eec4328ac7cb7499cfe4263] 2024-11-23T15:41:04,360 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:41:04,360 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): e50851c96ec6702e3c34ccb64f6cb9d4/B is initiating minor compaction (all files) 2024-11-23T15:41:04,360 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e9f6d11debb404ba5a3a29e85e23a34, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732376457974 2024-11-23T15:41:04,360 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e50851c96ec6702e3c34ccb64f6cb9d4/B in TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:04,360 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/82d17813839047998f0d8df1e68778b8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/6719e502ade244f08bcface8bc4e4ea8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/4f199fbdd48b4bd4b0180abf5ddfd167, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/ee188b5f17114571afbd555071682111] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp, totalSize=48.3 K 2024-11-23T15:41:04,361 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8469c0440bac429da17af741093c2309, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732376458648 2024-11-23T15:41:04,361 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 82d17813839047998f0d8df1e68778b8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732376457974 2024-11-23T15:41:04,361 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6719e502ade244f08bcface8bc4e4ea8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732376458648 2024-11-23T15:41:04,361 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba2380618c9c4d048eea84238c0943e8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732376460862 2024-11-23T15:41:04,361 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f199fbdd48b4bd4b0180abf5ddfd167, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732376460862 2024-11-23T15:41:04,361 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae7a2bb18eec4328ac7cb7499cfe4263, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732376463062 2024-11-23T15:41:04,361 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting ee188b5f17114571afbd555071682111, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732376463064 2024-11-23T15:41:04,368 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:41:04,369 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e50851c96ec6702e3c34ccb64f6cb9d4#B#compaction#180 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:04,370 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/212ec7175512419baccbdb7c2adc02e3 is 50, key is test_row_0/B:col10/1732376463070/Put/seqid=0 2024-11-23T15:41:04,370 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123b9e9c69b58244b26a9d782fe34d9042c_e50851c96ec6702e3c34ccb64f6cb9d4 store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:41:04,373 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123b9e9c69b58244b26a9d782fe34d9042c_e50851c96ec6702e3c34ccb64f6cb9d4, store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:41:04,374 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123b9e9c69b58244b26a9d782fe34d9042c_e50851c96ec6702e3c34ccb64f6cb9d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:41:04,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742041_1217 (size=4469) 2024-11-23T15:41:04,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742040_1216 (size=12983) 2024-11-23T15:41:04,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-23T15:41:04,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:04,414 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing e50851c96ec6702e3c34ccb64f6cb9d4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T15:41:04,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123b51e632784d245359d89673846eedecd_e50851c96ec6702e3c34ccb64f6cb9d4 is 50, key is test_row_0/A:col10/1732376463117/Put/seqid=0 2024-11-23T15:41:04,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742042_1218 (size=12454) 2024-11-23T15:41:04,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:04,431 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123b51e632784d245359d89673846eedecd_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b51e632784d245359d89673846eedecd_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:04,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/1d252b5bd5804d4ea37f3d9dde5da216, store: [table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:41:04,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/1d252b5bd5804d4ea37f3d9dde5da216 is 175, key is test_row_0/A:col10/1732376463117/Put/seqid=0 2024-11-23T15:41:04,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742043_1219 (size=31255) 2024-11-23T15:41:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-23T15:41:04,784 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e50851c96ec6702e3c34ccb64f6cb9d4#A#compaction#181 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:04,784 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/bd5cae2aa72449aa880d47746e64eeba is 175, key is test_row_0/A:col10/1732376463070/Put/seqid=0 2024-11-23T15:41:04,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742044_1220 (size=31937) 2024-11-23T15:41:04,797 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/212ec7175512419baccbdb7c2adc02e3 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/212ec7175512419baccbdb7c2adc02e3 2024-11-23T15:41:04,803 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e50851c96ec6702e3c34ccb64f6cb9d4/B of e50851c96ec6702e3c34ccb64f6cb9d4 into 212ec7175512419baccbdb7c2adc02e3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:04,803 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:41:04,803 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., storeName=e50851c96ec6702e3c34ccb64f6cb9d4/B, priority=12, startTime=1732376464358; duration=0sec 2024-11-23T15:41:04,803 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:04,803 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e50851c96ec6702e3c34ccb64f6cb9d4:B 2024-11-23T15:41:04,804 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:41:04,805 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:41:04,805 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): e50851c96ec6702e3c34ccb64f6cb9d4/C is initiating minor compaction (all files) 2024-11-23T15:41:04,805 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e50851c96ec6702e3c34ccb64f6cb9d4/C in TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:04,805 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/d09001c5f4a046c28fdab73950de70fb, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/686177469b5142eda682c96d4feeb327, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/e496ff4a643d47b08aeaeeafae97e2e7, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/cc11434381d44e57bf94818462a0de5c] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp, totalSize=48.3 K 2024-11-23T15:41:04,805 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting d09001c5f4a046c28fdab73950de70fb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732376457974 2024-11-23T15:41:04,806 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 686177469b5142eda682c96d4feeb327, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732376458648 2024-11-23T15:41:04,806 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting e496ff4a643d47b08aeaeeafae97e2e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732376460862 2024-11-23T15:41:04,806 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting cc11434381d44e57bf94818462a0de5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732376463064 2024-11-23T15:41:04,814 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e50851c96ec6702e3c34ccb64f6cb9d4#C#compaction#183 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:04,815 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/8a77592c339a4b30a237c309805378de is 50, key is test_row_0/C:col10/1732376463070/Put/seqid=0 2024-11-23T15:41:04,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742045_1221 (size=12983) 2024-11-23T15:41:04,837 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/8a77592c339a4b30a237c309805378de as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/8a77592c339a4b30a237c309805378de 2024-11-23T15:41:04,837 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/1d252b5bd5804d4ea37f3d9dde5da216 2024-11-23T15:41:04,845 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e50851c96ec6702e3c34ccb64f6cb9d4/C of e50851c96ec6702e3c34ccb64f6cb9d4 into 8a77592c339a4b30a237c309805378de(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:04,845 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:41:04,845 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., storeName=e50851c96ec6702e3c34ccb64f6cb9d4/C, priority=12, startTime=1732376464358; duration=0sec 2024-11-23T15:41:04,845 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:04,845 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e50851c96ec6702e3c34ccb64f6cb9d4:C 2024-11-23T15:41:04,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/13efddd564374e5fb2b59413d787e538 is 50, key is test_row_0/B:col10/1732376463117/Put/seqid=0 2024-11-23T15:41:04,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742046_1222 (size=12301) 2024-11-23T15:41:05,201 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/bd5cae2aa72449aa880d47746e64eeba as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/bd5cae2aa72449aa880d47746e64eeba 2024-11-23T15:41:05,218 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e50851c96ec6702e3c34ccb64f6cb9d4/A of e50851c96ec6702e3c34ccb64f6cb9d4 into bd5cae2aa72449aa880d47746e64eeba(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:05,218 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:41:05,218 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., storeName=e50851c96ec6702e3c34ccb64f6cb9d4/A, priority=12, startTime=1732376464358; duration=0sec 2024-11-23T15:41:05,218 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:05,218 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e50851c96ec6702e3c34ccb64f6cb9d4:A 2024-11-23T15:41:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:05,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. as already flushing 2024-11-23T15:41:05,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376525284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:05,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376525286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:05,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376525287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:05,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:05,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54566 deadline: 1732376525294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:05,295 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_2' in e50851c96ec6702e3c34ccb64f6cb9d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:41:05,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:05,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54606 deadline: 1732376525295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:05,297 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_1' in e50851c96ec6702e3c34ccb64f6cb9d4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:41:05,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:05,305 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/13efddd564374e5fb2b59413d787e538 2024-11-23T15:41:05,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/93ad26a8ab55421496f98ca4c58b84e5 is 50, key is test_row_0/C:col10/1732376463117/Put/seqid=0 2024-11-23T15:41:05,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742047_1223 (size=12301) 2024-11-23T15:41:05,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376525394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:05,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376525394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:05,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376525403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:05,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:05,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376525604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:05,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:05,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376525604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:05,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:05,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:05,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376525614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:05,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-23T15:41:05,724 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/93ad26a8ab55421496f98ca4c58b84e5 2024-11-23T15:41:05,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/1d252b5bd5804d4ea37f3d9dde5da216 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/1d252b5bd5804d4ea37f3d9dde5da216 2024-11-23T15:41:05,733 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/1d252b5bd5804d4ea37f3d9dde5da216, entries=150, sequenceid=313, filesize=30.5 K 2024-11-23T15:41:05,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/13efddd564374e5fb2b59413d787e538 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/13efddd564374e5fb2b59413d787e538 2024-11-23T15:41:05,738 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/13efddd564374e5fb2b59413d787e538, entries=150, sequenceid=313, filesize=12.0 K 2024-11-23T15:41:05,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/93ad26a8ab55421496f98ca4c58b84e5 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/93ad26a8ab55421496f98ca4c58b84e5 2024-11-23T15:41:05,743 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/93ad26a8ab55421496f98ca4c58b84e5, entries=150, sequenceid=313, filesize=12.0 K 2024-11-23T15:41:05,745 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e50851c96ec6702e3c34ccb64f6cb9d4 in 1331ms, sequenceid=313, compaction requested=false 2024-11-23T15:41:05,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:41:05,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:05,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-23T15:41:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-23T15:41:05,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-23T15:41:05,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2490 sec 2024-11-23T15:41:05,752 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.2540 sec 2024-11-23T15:41:05,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:05,918 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:41:05,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e50851c96ec6702e3c34ccb64f6cb9d4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T15:41:05,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411234acbc384d6294b08aae97a3e82e28d0b_e50851c96ec6702e3c34ccb64f6cb9d4 is 50, key is test_row_0/A:col10/1732376465276/Put/seqid=0 2024-11-23T15:41:05,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742048_1224 (size=14994) 2024-11-23T15:41:05,960 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:05,966 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411234acbc384d6294b08aae97a3e82e28d0b_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411234acbc384d6294b08aae97a3e82e28d0b_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:05,967 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/6a7afde9a9b34782ac860cd7b47a0021, store: [table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:41:05,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/6a7afde9a9b34782ac860cd7b47a0021 is 175, key is test_row_0/A:col10/1732376465276/Put/seqid=0 2024-11-23T15:41:05,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742049_1225 (size=39949) 2024-11-23T15:41:05,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376525984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:05,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376525985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:05,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376525984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:06,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376526094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:06,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376526095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:06,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376526095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:06,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:06,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:06,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:06,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:06,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:06,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:06,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:06,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:06,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:06,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376526304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:06,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:06,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376526304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:06,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376526304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:06,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:06,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:06,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:06,382 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/6a7afde9a9b34782ac860cd7b47a0021 2024-11-23T15:41:06,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/8a7e60ea1a7b4b47ab4867db3a0d875b is 50, key is test_row_0/B:col10/1732376465276/Put/seqid=0 2024-11-23T15:41:06,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742050_1226 (size=12301) 2024-11-23T15:41:06,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376526614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:06,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:06,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376526614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:06,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:06,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:06,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:06,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:06,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:06,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376526614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:06,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:06,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/8a7e60ea1a7b4b47ab4867db3a0d875b 2024-11-23T15:41:06,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/317e0cd193274588830eda5d1cf38e75 is 50, key is test_row_0/C:col10/1732376465276/Put/seqid=0 2024-11-23T15:41:06,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742051_1227 (size=12301) 2024-11-23T15:41:07,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54590 deadline: 1732376527124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:07,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:07,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:07,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54546 deadline: 1732376527124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:07,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:07,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:07,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:07,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54586 deadline: 1732376527124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:07,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:07,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:07,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:07,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:07,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/317e0cd193274588830eda5d1cf38e75 2024-11-23T15:41:07,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/6a7afde9a9b34782ac860cd7b47a0021 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/6a7afde9a9b34782ac860cd7b47a0021 2024-11-23T15:41:07,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/6a7afde9a9b34782ac860cd7b47a0021, entries=200, sequenceid=329, filesize=39.0 K 2024-11-23T15:41:07,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/8a7e60ea1a7b4b47ab4867db3a0d875b as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/8a7e60ea1a7b4b47ab4867db3a0d875b 2024-11-23T15:41:07,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/8a7e60ea1a7b4b47ab4867db3a0d875b, entries=150, sequenceid=329, filesize=12.0 K 2024-11-23T15:41:07,302 INFO [Thread-682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-682. 2024-11-23T15:41:07,302 INFO [Thread-686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-686. 2024-11-23T15:41:07,302 DEBUG [Thread-686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:41:07,302 DEBUG [Thread-682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:41:07,302 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:07,302 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:07,302 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:07,302 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:07,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:41:07,302 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:41:07,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/317e0cd193274588830eda5d1cf38e75 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/317e0cd193274588830eda5d1cf38e75 2024-11-23T15:41:07,305 INFO [Thread-684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-684. 2024-11-23T15:41:07,305 DEBUG [Thread-684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:41:07,305 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:07,305 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:07,305 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:41:07,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/317e0cd193274588830eda5d1cf38e75, entries=150, sequenceid=329, filesize=12.0 K 2024-11-23T15:41:07,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e50851c96ec6702e3c34ccb64f6cb9d4 in 1389ms, sequenceid=329, compaction requested=true 2024-11-23T15:41:07,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:41:07,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e50851c96ec6702e3c34ccb64f6cb9d4:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:41:07,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:07,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e50851c96ec6702e3c34ccb64f6cb9d4:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:41:07,307 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:07,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:07,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e50851c96ec6702e3c34ccb64f6cb9d4:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:41:07,308 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:07,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:07,308 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:07,308 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:07,309 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): e50851c96ec6702e3c34ccb64f6cb9d4/B is initiating minor compaction (all files) 2024-11-23T15:41:07,309 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): e50851c96ec6702e3c34ccb64f6cb9d4/A is initiating minor compaction (all files) 2024-11-23T15:41:07,309 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e50851c96ec6702e3c34ccb64f6cb9d4/A in TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:07,309 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e50851c96ec6702e3c34ccb64f6cb9d4/B in TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:07,309 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/212ec7175512419baccbdb7c2adc02e3, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/13efddd564374e5fb2b59413d787e538, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/8a7e60ea1a7b4b47ab4867db3a0d875b] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp, totalSize=36.7 K 2024-11-23T15:41:07,309 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/bd5cae2aa72449aa880d47746e64eeba, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/1d252b5bd5804d4ea37f3d9dde5da216, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/6a7afde9a9b34782ac860cd7b47a0021] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp, totalSize=100.7 K 2024-11-23T15:41:07,309 INFO [Thread-688 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-688. 2024-11-23T15:41:07,309 DEBUG [Thread-688 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:41:07,309 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:07,309 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:07,309 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. files: [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/bd5cae2aa72449aa880d47746e64eeba, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/1d252b5bd5804d4ea37f3d9dde5da216, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/6a7afde9a9b34782ac860cd7b47a0021] 2024-11-23T15:41:07,309 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:07,309 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:41:07,309 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 212ec7175512419baccbdb7c2adc02e3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732376463064 2024-11-23T15:41:07,309 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd5cae2aa72449aa880d47746e64eeba, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732376463064 2024-11-23T15:41:07,310 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 13efddd564374e5fb2b59413d787e538, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732376463117 2024-11-23T15:41:07,310 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d252b5bd5804d4ea37f3d9dde5da216, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732376463117 2024-11-23T15:41:07,310 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a7e60ea1a7b4b47ab4867db3a0d875b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732376465276 2024-11-23T15:41:07,310 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a7afde9a9b34782ac860cd7b47a0021, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732376465276 2024-11-23T15:41:07,315 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:41:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:07,316 INFO [Thread-673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-673. 2024-11-23T15:41:07,316 DEBUG [Thread-673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:41:07,316 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:07,316 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:07,316 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:41:07,316 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:41:07,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e50851c96ec6702e3c34ccb64f6cb9d4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T15:41:07,316 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e50851c96ec6702e3c34ccb64f6cb9d4#B#compaction#190 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:07,317 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/66608fb52f8648dabf609f42556d3a34 is 50, key is test_row_0/B:col10/1732376465276/Put/seqid=0 2024-11-23T15:41:07,317 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123f56fe3e7683e453181853a4fb19ea44c_e50851c96ec6702e3c34ccb64f6cb9d4 store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:41:07,319 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123f56fe3e7683e453181853a4fb19ea44c_e50851c96ec6702e3c34ccb64f6cb9d4, store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:41:07,319 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123f56fe3e7683e453181853a4fb19ea44c_e50851c96ec6702e3c34ccb64f6cb9d4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:41:07,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742052_1228 (size=13085) 2024-11-23T15:41:07,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123db8e704a56e14f12a7d2d87a08a685f2_e50851c96ec6702e3c34ccb64f6cb9d4 is 50, key is test_row_0/A:col10/1732376465983/Put/seqid=0 2024-11-23T15:41:07,325 INFO [Thread-671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-671. 2024-11-23T15:41:07,325 DEBUG [Thread-671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:41:07,325 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:07,325 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:07,325 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:41:07,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742053_1229 (size=4469) 2024-11-23T15:41:07,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742054_1230 (size=12454) 2024-11-23T15:41:07,450 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T15:41:07,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-23T15:41:07,655 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:07,729 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e50851c96ec6702e3c34ccb64f6cb9d4#A#compaction#189 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:07,730 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/3ba11e07034847f6a8725e391d343245 is 175, key is test_row_0/A:col10/1732376465276/Put/seqid=0 2024-11-23T15:41:07,730 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:07,731 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/66608fb52f8648dabf609f42556d3a34 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/66608fb52f8648dabf609f42556d3a34 2024-11-23T15:41:07,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742055_1231 (size=32039) 2024-11-23T15:41:07,736 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123db8e704a56e14f12a7d2d87a08a685f2_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123db8e704a56e14f12a7d2d87a08a685f2_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:07,737 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/a4963661a2884e749eea0c3d2ca0995b, store: [table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:41:07,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/a4963661a2884e749eea0c3d2ca0995b is 175, key is test_row_0/A:col10/1732376465983/Put/seqid=0 2024-11-23T15:41:07,738 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e50851c96ec6702e3c34ccb64f6cb9d4/B of e50851c96ec6702e3c34ccb64f6cb9d4 into 66608fb52f8648dabf609f42556d3a34(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:07,738 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:41:07,738 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., storeName=e50851c96ec6702e3c34ccb64f6cb9d4/B, priority=13, startTime=1732376467307; duration=0sec 2024-11-23T15:41:07,738 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:07,738 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e50851c96ec6702e3c34ccb64f6cb9d4:B 2024-11-23T15:41:07,739 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:07,740 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:07,740 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): e50851c96ec6702e3c34ccb64f6cb9d4/C is initiating minor compaction (all files) 2024-11-23T15:41:07,740 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e50851c96ec6702e3c34ccb64f6cb9d4/C in TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:07,740 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/8a77592c339a4b30a237c309805378de, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/93ad26a8ab55421496f98ca4c58b84e5, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/317e0cd193274588830eda5d1cf38e75] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp, totalSize=36.7 K 2024-11-23T15:41:07,740 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a77592c339a4b30a237c309805378de, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732376463064 2024-11-23T15:41:07,741 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 93ad26a8ab55421496f98ca4c58b84e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732376463117 2024-11-23T15:41:07,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742056_1232 (size=31255) 2024-11-23T15:41:07,741 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 317e0cd193274588830eda5d1cf38e75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732376465276 2024-11-23T15:41:07,742 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/a4963661a2884e749eea0c3d2ca0995b 2024-11-23T15:41:07,742 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/3ba11e07034847f6a8725e391d343245 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/3ba11e07034847f6a8725e391d343245 2024-11-23T15:41:07,747 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e50851c96ec6702e3c34ccb64f6cb9d4/A of e50851c96ec6702e3c34ccb64f6cb9d4 into 3ba11e07034847f6a8725e391d343245(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:07,747 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:41:07,747 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., storeName=e50851c96ec6702e3c34ccb64f6cb9d4/A, priority=13, startTime=1732376467307; duration=0sec 2024-11-23T15:41:07,747 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:07,747 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e50851c96ec6702e3c34ccb64f6cb9d4:A 2024-11-23T15:41:07,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/235ebbcc804c494da55f5c892a10e33a is 50, key is test_row_0/B:col10/1732376465983/Put/seqid=0 2024-11-23T15:41:07,747 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e50851c96ec6702e3c34ccb64f6cb9d4#C#compaction#193 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:07,748 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/6b7cf1d33791461c97178df44f79ff73 is 50, key is test_row_0/C:col10/1732376465276/Put/seqid=0 2024-11-23T15:41:07,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742057_1233 (size=12301) 2024-11-23T15:41:07,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742058_1234 (size=13085) 2024-11-23T15:41:08,136 INFO [Thread-675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-675. 2024-11-23T15:41:08,136 INFO [Thread-679 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-679. 2024-11-23T15:41:08,136 INFO [Thread-677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-677. 2024-11-23T15:41:08,136 DEBUG [Thread-675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:41:08,136 DEBUG [Thread-679 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:41:08,136 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:08,136 DEBUG [Thread-679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:08,136 DEBUG [Thread-677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:41:08,136 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:08,136 DEBUG [Thread-679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:08,136 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:08,136 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:41:08,136 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:41:08,136 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:08,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T15:41:08,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-23T15:41:08,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-23T15:41:08,136 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:41:08,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-11-23T15:41:08,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-23T15:41:08,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-23T15:41:08,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T15:41:08,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3692 2024-11-23T15:41:08,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3628 2024-11-23T15:41:08,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T15:41:08,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1649 2024-11-23T15:41:08,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4946 rows 2024-11-23T15:41:08,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1641 2024-11-23T15:41:08,137 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4920 rows 2024-11-23T15:41:08,137 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T15:41:08,137 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T15:41:08,137 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:08,138 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:08,138 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:41:08,139 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-23T15:41:08,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T15:41:08,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-23T15:41:08,145 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376468144"}]},"ts":"1732376468144"} 2024-11-23T15:41:08,147 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T15:41:08,147 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T15:41:08,148 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T15:41:08,149 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e50851c96ec6702e3c34ccb64f6cb9d4, UNASSIGN}] 2024-11-23T15:41:08,149 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e50851c96ec6702e3c34ccb64f6cb9d4, UNASSIGN 2024-11-23T15:41:08,150 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=e50851c96ec6702e3c34ccb64f6cb9d4, regionState=CLOSING, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:08,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/235ebbcc804c494da55f5c892a10e33a 2024-11-23T15:41:08,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e50851c96ec6702e3c34ccb64f6cb9d4, UNASSIGN because future has completed 2024-11-23T15:41:08,152 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T15:41:08,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194}] 2024-11-23T15:41:08,156 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/6b7cf1d33791461c97178df44f79ff73 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/6b7cf1d33791461c97178df44f79ff73 2024-11-23T15:41:08,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/eee8a20340d544c6b7c423383d988806 is 50, key is test_row_0/C:col10/1732376465983/Put/seqid=0 2024-11-23T15:41:08,161 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e50851c96ec6702e3c34ccb64f6cb9d4/C of e50851c96ec6702e3c34ccb64f6cb9d4 into 6b7cf1d33791461c97178df44f79ff73(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:08,161 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:41:08,161 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., storeName=e50851c96ec6702e3c34ccb64f6cb9d4/C, priority=13, startTime=1732376467307; duration=0sec 2024-11-23T15:41:08,161 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:08,161 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e50851c96ec6702e3c34ccb64f6cb9d4:C 2024-11-23T15:41:08,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742059_1235 (size=12301) 2024-11-23T15:41:08,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-23T15:41:08,305 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:08,305 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T15:41:08,305 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing e50851c96ec6702e3c34ccb64f6cb9d4, disabling compactions & flushes 2024-11-23T15:41:08,306 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:08,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-23T15:41:08,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/eee8a20340d544c6b7c423383d988806 2024-11-23T15:41:08,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/a4963661a2884e749eea0c3d2ca0995b as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/a4963661a2884e749eea0c3d2ca0995b 2024-11-23T15:41:08,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/a4963661a2884e749eea0c3d2ca0995b, entries=150, sequenceid=353, filesize=30.5 K 2024-11-23T15:41:08,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/235ebbcc804c494da55f5c892a10e33a as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/235ebbcc804c494da55f5c892a10e33a 2024-11-23T15:41:08,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/235ebbcc804c494da55f5c892a10e33a, entries=150, sequenceid=353, filesize=12.0 K 2024-11-23T15:41:08,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/eee8a20340d544c6b7c423383d988806 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/eee8a20340d544c6b7c423383d988806 2024-11-23T15:41:08,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/eee8a20340d544c6b7c423383d988806, entries=150, sequenceid=353, filesize=12.0 K 2024-11-23T15:41:08,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=26.84 KB/27480 for e50851c96ec6702e3c34ccb64f6cb9d4 in 1264ms, sequenceid=353, compaction requested=false 2024-11-23T15:41:08,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e50851c96ec6702e3c34ccb64f6cb9d4: 2024-11-23T15:41:08,580 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:08,580 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:08,580 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. after waiting 0 ms 2024-11-23T15:41:08,580 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:08,580 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing e50851c96ec6702e3c34ccb64f6cb9d4 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-23T15:41:08,585 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123051243f3b2914445aa0f8fe63ce3d5c6_e50851c96ec6702e3c34ccb64f6cb9d4 is 50, key is test_row_1/A:col10/1732376467324/Put/seqid=0 2024-11-23T15:41:08,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742060_1236 (size=9914) 2024-11-23T15:41:08,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-23T15:41:08,992 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:09,003 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123051243f3b2914445aa0f8fe63ce3d5c6_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123051243f3b2914445aa0f8fe63ce3d5c6_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:09,004 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/4d56ec0694904b42a911e47fae25febd, store: [table=TestAcidGuarantees family=A region=e50851c96ec6702e3c34ccb64f6cb9d4] 2024-11-23T15:41:09,005 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/4d56ec0694904b42a911e47fae25febd is 175, key is test_row_1/A:col10/1732376467324/Put/seqid=0 2024-11-23T15:41:09,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742061_1237 (size=22561) 2024-11-23T15:41:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-23T15:41:09,411 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=363, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/4d56ec0694904b42a911e47fae25febd 2024-11-23T15:41:09,425 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/b1d58fe298ef4720b8b7c459879cdc5a is 50, key is test_row_1/B:col10/1732376467324/Put/seqid=0 2024-11-23T15:41:09,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742062_1238 (size=9857) 2024-11-23T15:41:09,831 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/b1d58fe298ef4720b8b7c459879cdc5a 2024-11-23T15:41:09,845 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/ffa298c111a3473a812492527c05ad6d is 50, key is test_row_1/C:col10/1732376467324/Put/seqid=0 2024-11-23T15:41:09,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742063_1239 (size=9857) 2024-11-23T15:41:10,251 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/ffa298c111a3473a812492527c05ad6d 2024-11-23T15:41:10,263 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/A/4d56ec0694904b42a911e47fae25febd as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/4d56ec0694904b42a911e47fae25febd 2024-11-23T15:41:10,266 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/4d56ec0694904b42a911e47fae25febd, entries=100, sequenceid=363, filesize=22.0 K 2024-11-23T15:41:10,267 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/B/b1d58fe298ef4720b8b7c459879cdc5a as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/b1d58fe298ef4720b8b7c459879cdc5a 2024-11-23T15:41:10,270 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/b1d58fe298ef4720b8b7c459879cdc5a, entries=100, sequenceid=363, filesize=9.6 K 2024-11-23T15:41:10,271 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/.tmp/C/ffa298c111a3473a812492527c05ad6d as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/ffa298c111a3473a812492527c05ad6d 2024-11-23T15:41:10,274 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/ffa298c111a3473a812492527c05ad6d, entries=100, sequenceid=363, filesize=9.6 K 2024-11-23T15:41:10,275 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for e50851c96ec6702e3c34ccb64f6cb9d4 in 1695ms, sequenceid=363, compaction requested=true 2024-11-23T15:41:10,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/bff9ef12b9144589921021f713701dc9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ead4bc2f89164c78adab6f60b173986d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/566e1e6c33234f09b9c7afe5a2294bd8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/e2282c8c965a4159b74ea73534699c8c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ccb85e09e6a748dd9b42f2a437eb34fb, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/028cb4a96bb54af48231f7e92758b467, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/0adaaee5fd014407ba3a87a3e48e5dd4, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/6c5e18df32aa485bbfd8e8d4ab8b3ac3, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/e62c6e1f745b4d40b191cb775b99f80a, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/f7050ecec00a4ada959e21d51a96c4f4, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/8e91bf36582248818b380190eae63eee, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/10e76ea5e84c4943ae8991bef94b978c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/5c61a949652648aeb32f1297b0acef40, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/1a37d920bd034953a37a9c319ef06e4d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/a5b93532a4f2414cad04b62f86e8ddab, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/d63f373292ca489b8ba9c646b7aa280c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/0e9f6d11debb404ba5a3a29e85e23a34, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/8469c0440bac429da17af741093c2309, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ba2380618c9c4d048eea84238c0943e8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ae7a2bb18eec4328ac7cb7499cfe4263, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/bd5cae2aa72449aa880d47746e64eeba, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/1d252b5bd5804d4ea37f3d9dde5da216, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/6a7afde9a9b34782ac860cd7b47a0021] to archive 2024-11-23T15:41:10,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T15:41:10,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/bff9ef12b9144589921021f713701dc9 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/bff9ef12b9144589921021f713701dc9 2024-11-23T15:41:10,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ead4bc2f89164c78adab6f60b173986d to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ead4bc2f89164c78adab6f60b173986d 2024-11-23T15:41:10,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/566e1e6c33234f09b9c7afe5a2294bd8 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/566e1e6c33234f09b9c7afe5a2294bd8 2024-11-23T15:41:10,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/e2282c8c965a4159b74ea73534699c8c to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/e2282c8c965a4159b74ea73534699c8c 2024-11-23T15:41:10,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ccb85e09e6a748dd9b42f2a437eb34fb to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ccb85e09e6a748dd9b42f2a437eb34fb 2024-11-23T15:41:10,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/028cb4a96bb54af48231f7e92758b467 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/028cb4a96bb54af48231f7e92758b467 2024-11-23T15:41:10,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/0adaaee5fd014407ba3a87a3e48e5dd4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/0adaaee5fd014407ba3a87a3e48e5dd4 2024-11-23T15:41:10,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/6c5e18df32aa485bbfd8e8d4ab8b3ac3 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/6c5e18df32aa485bbfd8e8d4ab8b3ac3 2024-11-23T15:41:10,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/e62c6e1f745b4d40b191cb775b99f80a to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/e62c6e1f745b4d40b191cb775b99f80a 2024-11-23T15:41:10,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/f7050ecec00a4ada959e21d51a96c4f4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/f7050ecec00a4ada959e21d51a96c4f4 2024-11-23T15:41:10,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/8e91bf36582248818b380190eae63eee to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/8e91bf36582248818b380190eae63eee 2024-11-23T15:41:10,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/10e76ea5e84c4943ae8991bef94b978c to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/10e76ea5e84c4943ae8991bef94b978c 2024-11-23T15:41:10,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/5c61a949652648aeb32f1297b0acef40 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/5c61a949652648aeb32f1297b0acef40 2024-11-23T15:41:10,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/1a37d920bd034953a37a9c319ef06e4d to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/1a37d920bd034953a37a9c319ef06e4d 2024-11-23T15:41:10,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/a5b93532a4f2414cad04b62f86e8ddab to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/a5b93532a4f2414cad04b62f86e8ddab 2024-11-23T15:41:10,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/d63f373292ca489b8ba9c646b7aa280c to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/d63f373292ca489b8ba9c646b7aa280c 2024-11-23T15:41:10,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/0e9f6d11debb404ba5a3a29e85e23a34 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/0e9f6d11debb404ba5a3a29e85e23a34 2024-11-23T15:41:10,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/8469c0440bac429da17af741093c2309 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/8469c0440bac429da17af741093c2309 2024-11-23T15:41:10,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ba2380618c9c4d048eea84238c0943e8 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ba2380618c9c4d048eea84238c0943e8 2024-11-23T15:41:10,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ae7a2bb18eec4328ac7cb7499cfe4263 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/ae7a2bb18eec4328ac7cb7499cfe4263 2024-11-23T15:41:10,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/bd5cae2aa72449aa880d47746e64eeba to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/bd5cae2aa72449aa880d47746e64eeba 2024-11-23T15:41:10,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/1d252b5bd5804d4ea37f3d9dde5da216 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/1d252b5bd5804d4ea37f3d9dde5da216 2024-11-23T15:41:10,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/6a7afde9a9b34782ac860cd7b47a0021 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/6a7afde9a9b34782ac860cd7b47a0021 2024-11-23T15:41:10,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/0acd431422014fb0877f95a9abaa8d86, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/d0a640ef2a3e41258109cc4fd35f1e87, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/b651d8c1ed0647389e2c5f04cc3c94bc, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/2bad4fda470341b1a2bb5ccbc8d20e2f, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/2dad067be64842d3b103442711ef62aa, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/35c532aa098d43fa80fa09c73f52f893, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/f77b6a7c93da4948b408c98976dff510, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/d27b474a3ed34f80aa3b584344d9cb23, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/404f2cc02d3a4642b1eed9bd0553b4ef, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/dc226e0965474e108d7f25c4feeee487, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/44abd19c53b341b5962346b279906af8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/0705e430eddf42a2b456e221b241bda6, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/d8a305e267164f8c9084162174ca2658, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/65744128ecaf420fb7fc36b28bcddebb, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/4662961ff57e41bb9228eae41c84aad0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/82d17813839047998f0d8df1e68778b8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/f46f888bedbc43758f7cbe55d77dd328, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/6719e502ade244f08bcface8bc4e4ea8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/4f199fbdd48b4bd4b0180abf5ddfd167, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/212ec7175512419baccbdb7c2adc02e3, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/ee188b5f17114571afbd555071682111, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/13efddd564374e5fb2b59413d787e538, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/8a7e60ea1a7b4b47ab4867db3a0d875b] to archive 2024-11-23T15:41:10,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T15:41:10,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/0acd431422014fb0877f95a9abaa8d86 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/0acd431422014fb0877f95a9abaa8d86 2024-11-23T15:41:10,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/d0a640ef2a3e41258109cc4fd35f1e87 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/d0a640ef2a3e41258109cc4fd35f1e87 2024-11-23T15:41:10,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/b651d8c1ed0647389e2c5f04cc3c94bc to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/b651d8c1ed0647389e2c5f04cc3c94bc 2024-11-23T15:41:10,303 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/2bad4fda470341b1a2bb5ccbc8d20e2f to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/2bad4fda470341b1a2bb5ccbc8d20e2f 2024-11-23T15:41:10,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/2dad067be64842d3b103442711ef62aa to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/2dad067be64842d3b103442711ef62aa 2024-11-23T15:41:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-23T15:41:10,305 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/35c532aa098d43fa80fa09c73f52f893 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/35c532aa098d43fa80fa09c73f52f893 2024-11-23T15:41:10,305 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/f77b6a7c93da4948b408c98976dff510 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/f77b6a7c93da4948b408c98976dff510 2024-11-23T15:41:10,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/d27b474a3ed34f80aa3b584344d9cb23 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/d27b474a3ed34f80aa3b584344d9cb23 2024-11-23T15:41:10,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/404f2cc02d3a4642b1eed9bd0553b4ef to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/404f2cc02d3a4642b1eed9bd0553b4ef 2024-11-23T15:41:10,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/dc226e0965474e108d7f25c4feeee487 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/dc226e0965474e108d7f25c4feeee487 2024-11-23T15:41:10,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/44abd19c53b341b5962346b279906af8 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/44abd19c53b341b5962346b279906af8 2024-11-23T15:41:10,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/0705e430eddf42a2b456e221b241bda6 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/0705e430eddf42a2b456e221b241bda6 2024-11-23T15:41:10,310 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/d8a305e267164f8c9084162174ca2658 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/d8a305e267164f8c9084162174ca2658 2024-11-23T15:41:10,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/65744128ecaf420fb7fc36b28bcddebb to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/65744128ecaf420fb7fc36b28bcddebb 2024-11-23T15:41:10,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/4662961ff57e41bb9228eae41c84aad0 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/4662961ff57e41bb9228eae41c84aad0 2024-11-23T15:41:10,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/82d17813839047998f0d8df1e68778b8 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/82d17813839047998f0d8df1e68778b8 2024-11-23T15:41:10,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/f46f888bedbc43758f7cbe55d77dd328 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/f46f888bedbc43758f7cbe55d77dd328 2024-11-23T15:41:10,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/6719e502ade244f08bcface8bc4e4ea8 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/6719e502ade244f08bcface8bc4e4ea8 2024-11-23T15:41:10,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/4f199fbdd48b4bd4b0180abf5ddfd167 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/4f199fbdd48b4bd4b0180abf5ddfd167 2024-11-23T15:41:10,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/212ec7175512419baccbdb7c2adc02e3 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/212ec7175512419baccbdb7c2adc02e3 2024-11-23T15:41:10,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/ee188b5f17114571afbd555071682111 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/ee188b5f17114571afbd555071682111 2024-11-23T15:41:10,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/13efddd564374e5fb2b59413d787e538 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/13efddd564374e5fb2b59413d787e538 2024-11-23T15:41:10,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/8a7e60ea1a7b4b47ab4867db3a0d875b to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/8a7e60ea1a7b4b47ab4867db3a0d875b 2024-11-23T15:41:10,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/14ef2cc565aa484595f924054f9c0950, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/ce54fc44f3824f8c97bfd381c3cb1dcd, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/7514975ad0234805af29366434160f45, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/8f14a41a2a484c2a8e00bd55b3d62c00, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/6122f533d307475ba416a422219077e4, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/8be98f63542a4c1c9032cca088fbe977, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/f47d234622e64f6ab9b1109e0af38e04, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/0fa4cbf7e230457cba0c355d69556208, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/917844199fe04366827d258e7dc33f59, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/6d14f2df26a54bff8014fc23cf527dff, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/8cc2c921b5014d9d8ac941ff7fb15de3, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/a5c56ac811654afb9e83c3d556d51e75, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/f99dd901d53c409a8e06aa9b04c530ad, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/92d78ad0743442eab084cd0f9e12e07a, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/f1c40746ef4c4b62917b141285bd1949, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/d09001c5f4a046c28fdab73950de70fb, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/1ce21fb735024a7eac7f9b608331bbc6, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/686177469b5142eda682c96d4feeb327, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/e496ff4a643d47b08aeaeeafae97e2e7, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/8a77592c339a4b30a237c309805378de, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/cc11434381d44e57bf94818462a0de5c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/93ad26a8ab55421496f98ca4c58b84e5, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/317e0cd193274588830eda5d1cf38e75] to archive 2024-11-23T15:41:10,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T15:41:10,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/14ef2cc565aa484595f924054f9c0950 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/14ef2cc565aa484595f924054f9c0950 2024-11-23T15:41:10,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/ce54fc44f3824f8c97bfd381c3cb1dcd to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/ce54fc44f3824f8c97bfd381c3cb1dcd 2024-11-23T15:41:10,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/7514975ad0234805af29366434160f45 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/7514975ad0234805af29366434160f45 2024-11-23T15:41:10,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/8f14a41a2a484c2a8e00bd55b3d62c00 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/8f14a41a2a484c2a8e00bd55b3d62c00 2024-11-23T15:41:10,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/6122f533d307475ba416a422219077e4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/6122f533d307475ba416a422219077e4 2024-11-23T15:41:10,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/8be98f63542a4c1c9032cca088fbe977 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/8be98f63542a4c1c9032cca088fbe977 2024-11-23T15:41:10,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/f47d234622e64f6ab9b1109e0af38e04 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/f47d234622e64f6ab9b1109e0af38e04 2024-11-23T15:41:10,326 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/0fa4cbf7e230457cba0c355d69556208 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/0fa4cbf7e230457cba0c355d69556208 2024-11-23T15:41:10,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/917844199fe04366827d258e7dc33f59 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/917844199fe04366827d258e7dc33f59 2024-11-23T15:41:10,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/6d14f2df26a54bff8014fc23cf527dff to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/6d14f2df26a54bff8014fc23cf527dff 2024-11-23T15:41:10,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/8cc2c921b5014d9d8ac941ff7fb15de3 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/8cc2c921b5014d9d8ac941ff7fb15de3 2024-11-23T15:41:10,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/a5c56ac811654afb9e83c3d556d51e75 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/a5c56ac811654afb9e83c3d556d51e75 2024-11-23T15:41:10,330 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/f99dd901d53c409a8e06aa9b04c530ad to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/f99dd901d53c409a8e06aa9b04c530ad 2024-11-23T15:41:10,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/92d78ad0743442eab084cd0f9e12e07a to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/92d78ad0743442eab084cd0f9e12e07a 2024-11-23T15:41:10,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/f1c40746ef4c4b62917b141285bd1949 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/f1c40746ef4c4b62917b141285bd1949 2024-11-23T15:41:10,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/d09001c5f4a046c28fdab73950de70fb to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/d09001c5f4a046c28fdab73950de70fb 2024-11-23T15:41:10,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/1ce21fb735024a7eac7f9b608331bbc6 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/1ce21fb735024a7eac7f9b608331bbc6 2024-11-23T15:41:10,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/686177469b5142eda682c96d4feeb327 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/686177469b5142eda682c96d4feeb327 2024-11-23T15:41:10,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/e496ff4a643d47b08aeaeeafae97e2e7 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/e496ff4a643d47b08aeaeeafae97e2e7 2024-11-23T15:41:10,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/8a77592c339a4b30a237c309805378de to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/8a77592c339a4b30a237c309805378de 2024-11-23T15:41:10,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/cc11434381d44e57bf94818462a0de5c to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/cc11434381d44e57bf94818462a0de5c 2024-11-23T15:41:10,336 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/93ad26a8ab55421496f98ca4c58b84e5 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/93ad26a8ab55421496f98ca4c58b84e5 2024-11-23T15:41:10,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/317e0cd193274588830eda5d1cf38e75 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/317e0cd193274588830eda5d1cf38e75 2024-11-23T15:41:10,340 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/recovered.edits/366.seqid, newMaxSeqId=366, maxSeqId=4 2024-11-23T15:41:10,341 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. 2024-11-23T15:41:10,341 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for e50851c96ec6702e3c34ccb64f6cb9d4: Waiting for close lock at 1732376468305Running coprocessor pre-close hooks at 1732376468305Disabling compacts and flushes for region at 1732376468305Disabling writes for close at 1732376468580 (+275 ms)Obtaining lock to block concurrent updates at 1732376468580Preparing flush snapshotting stores in e50851c96ec6702e3c34ccb64f6cb9d4 at 1732376468580Finished memstore snapshotting TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732376468580Flushing stores of TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4. at 1732376468581 (+1 ms)Flushing e50851c96ec6702e3c34ccb64f6cb9d4/A: creating writer at 1732376468581Flushing mob file e50851c96ec6702e3c34ccb64f6cb9d4/A: appending metadata at 1732376468585 (+4 ms)Flushing mob file e50851c96ec6702e3c34ccb64f6cb9d4/A: closing flushed file at 1732376468585Flushing e50851c96ec6702e3c34ccb64f6cb9d4/A: appending metadata at 1732376469004 (+419 ms)Flushing e50851c96ec6702e3c34ccb64f6cb9d4/A: closing flushed file at 1732376469004Flushing e50851c96ec6702e3c34ccb64f6cb9d4/B: creating writer at 1732376469421 (+417 ms)Flushing e50851c96ec6702e3c34ccb64f6cb9d4/B: appending metadata at 1732376469425 (+4 ms)Flushing e50851c96ec6702e3c34ccb64f6cb9d4/B: closing flushed file at 1732376469425Flushing e50851c96ec6702e3c34ccb64f6cb9d4/C: creating writer at 1732376469842 (+417 ms)Flushing e50851c96ec6702e3c34ccb64f6cb9d4/C: appending metadata at 1732376469845 (+3 ms)Flushing e50851c96ec6702e3c34ccb64f6cb9d4/C: closing flushed file at 1732376469845Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f179aca: reopening flushed file at 1732376470262 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ba31791: reopening flushed file at 1732376470266 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@682ed175: reopening flushed file at 1732376470270 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for e50851c96ec6702e3c34ccb64f6cb9d4 in 1695ms, sequenceid=363, compaction requested=true at 1732376470275 (+5 ms)Writing region close event to WAL at 1732376470338 (+63 ms)Running coprocessor post-close hooks at 1732376470341 (+3 ms)Closed at 1732376470341 2024-11-23T15:41:10,343 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:10,343 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=e50851c96ec6702e3c34ccb64f6cb9d4, regionState=CLOSED 2024-11-23T15:41:10,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 because future has completed 2024-11-23T15:41:10,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-23T15:41:10,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure e50851c96ec6702e3c34ccb64f6cb9d4, server=b712f9af2c12,46589,1732376409194 in 2.1930 sec 2024-11-23T15:41:10,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-23T15:41:10,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e50851c96ec6702e3c34ccb64f6cb9d4, UNASSIGN in 2.1970 sec 2024-11-23T15:41:10,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-23T15:41:10,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.2000 sec 2024-11-23T15:41:10,351 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376470351"}]},"ts":"1732376470351"} 2024-11-23T15:41:10,352 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T15:41:10,352 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T15:41:10,354 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.2140 sec 2024-11-23T15:41:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-23T15:41:12,324 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:12,325 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-23T15:41:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:41:12,327 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:41:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-23T15:41:12,327 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=66, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:41:12,330 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:12,333 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A, FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B, FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C, FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/recovered.edits] 2024-11-23T15:41:12,371 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/3ba11e07034847f6a8725e391d343245 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/3ba11e07034847f6a8725e391d343245 2024-11-23T15:41:12,373 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/4d56ec0694904b42a911e47fae25febd to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/4d56ec0694904b42a911e47fae25febd 2024-11-23T15:41:12,375 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/a4963661a2884e749eea0c3d2ca0995b to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/A/a4963661a2884e749eea0c3d2ca0995b 2024-11-23T15:41:12,378 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/235ebbcc804c494da55f5c892a10e33a to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/235ebbcc804c494da55f5c892a10e33a 2024-11-23T15:41:12,380 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/66608fb52f8648dabf609f42556d3a34 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/66608fb52f8648dabf609f42556d3a34 2024-11-23T15:41:12,381 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/b1d58fe298ef4720b8b7c459879cdc5a to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/B/b1d58fe298ef4720b8b7c459879cdc5a 2024-11-23T15:41:12,384 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/6b7cf1d33791461c97178df44f79ff73 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/6b7cf1d33791461c97178df44f79ff73 2024-11-23T15:41:12,386 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/eee8a20340d544c6b7c423383d988806 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/eee8a20340d544c6b7c423383d988806 2024-11-23T15:41:12,387 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/ffa298c111a3473a812492527c05ad6d to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/C/ffa298c111a3473a812492527c05ad6d 2024-11-23T15:41:12,391 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/recovered.edits/366.seqid to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4/recovered.edits/366.seqid 2024-11-23T15:41:12,391 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:12,392 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T15:41:12,392 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T15:41:12,393 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-23T15:41:12,397 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123051243f3b2914445aa0f8fe63ce3d5c6_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123051243f3b2914445aa0f8fe63ce3d5c6_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:12,399 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112327b2f3b1edec412396308e98b0ee3081_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112327b2f3b1edec412396308e98b0ee3081_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:12,400 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232dbdb353c89b4caeacec356ab4820dab_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232dbdb353c89b4caeacec356ab4820dab_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:12,401 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123351b52f528374a1c80177f185d44f630_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123351b52f528374a1c80177f185d44f630_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:12,402 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411234acbc384d6294b08aae97a3e82e28d0b_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411234acbc384d6294b08aae97a3e82e28d0b_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:12,403 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123596eef4ab3b8494da5757aa481f1fbdf_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123596eef4ab3b8494da5757aa481f1fbdf_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:12,404 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235c7519f595764664b79673b46f27e1f1_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235c7519f595764664b79673b46f27e1f1_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:12,406 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123a144ba639d124d10ba713f9dcb4be15f_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123a144ba639d124d10ba713f9dcb4be15f_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:12,407 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b1e0ad1a6ee544ed91f0cf88d9cb6077_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b1e0ad1a6ee544ed91f0cf88d9cb6077_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:12,408 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b51e632784d245359d89673846eedecd_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b51e632784d245359d89673846eedecd_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:12,410 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c021a6ea36fd4344818650456c0e2561_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c021a6ea36fd4344818650456c0e2561_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:12,411 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c1325903a70e4e8a988cb396008be669_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c1325903a70e4e8a988cb396008be669_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:12,412 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c7a69efe73c3473ca20747c9bac88040_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c7a69efe73c3473ca20747c9bac88040_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:12,414 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c9bf39a5c57c40d69564ed132d9948ea_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c9bf39a5c57c40d69564ed132d9948ea_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:12,415 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123d656681f401b41e58136f9e422b9913b_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123d656681f401b41e58136f9e422b9913b_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:12,416 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123d6f8270189ba4f479277e0287c617466_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123d6f8270189ba4f479277e0287c617466_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:12,418 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123db8e704a56e14f12a7d2d87a08a685f2_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123db8e704a56e14f12a7d2d87a08a685f2_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:12,419 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123deb0efc043374943a4ad4432172e3a4c_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123deb0efc043374943a4ad4432172e3a4c_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:12,420 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123fdb8ee3835d44e1295af36389ffceddd_e50851c96ec6702e3c34ccb64f6cb9d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123fdb8ee3835d44e1295af36389ffceddd_e50851c96ec6702e3c34ccb64f6cb9d4 2024-11-23T15:41:12,421 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T15:41:12,424 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=66, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:41:12,427 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T15:41:12,429 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T15:41:12,431 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=66, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:41:12,431 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T15:41:12,431 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732376472431"}]},"ts":"9223372036854775807"} 2024-11-23T15:41:12,433 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T15:41:12,433 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e50851c96ec6702e3c34ccb64f6cb9d4, NAME => 'TestAcidGuarantees,,1732376445089.e50851c96ec6702e3c34ccb64f6cb9d4.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T15:41:12,433 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T15:41:12,433 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732376472433"}]},"ts":"9223372036854775807"} 2024-11-23T15:41:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-23T15:41:12,435 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T15:41:12,436 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=66, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:41:12,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 111 msec 2024-11-23T15:41:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-23T15:41:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T15:41:12,645 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:12,661 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=242 (was 237) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1089955776_22 at /127.0.0.1:42560 [Waiting for operation #117] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1438786559_22 at /127.0.0.1:42552 [Waiting for operation #123] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1438786559_22 at /127.0.0.1:56940 [Waiting for operation #297] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/cluster_bcb5f073-d146-1328-644a-da8d77accbc4/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/cluster_bcb5f073-d146-1328-644a-da8d77accbc4/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1089955776_22 at /127.0.0.1:56860 [Waiting for operation #343] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=417 (was 408) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7223 (was 7469) 2024-11-23T15:41:12,669 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=417, ProcessCount=11, AvailableMemoryMB=7222 2024-11-23T15:41:12,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T15:41:12,670 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:41:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T15:41:12,672 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T15:41:12,673 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:12,673 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 67 2024-11-23T15:41:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-23T15:41:12,673 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T15:41:12,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742064_1240 (size=959) 2024-11-23T15:41:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-23T15:41:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-23T15:41:13,086 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e 2024-11-23T15:41:13,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742065_1241 (size=53) 2024-11-23T15:41:13,096 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:41:13,096 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d2607dde3b91c49826fdae09a26d7b59, disabling compactions & flushes 2024-11-23T15:41:13,096 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:13,096 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:13,096 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. after waiting 0 ms 2024-11-23T15:41:13,096 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:13,096 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:13,096 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d2607dde3b91c49826fdae09a26d7b59: Waiting for close lock at 1732376473096Disabling compacts and flushes for region at 1732376473096Disabling writes for close at 1732376473096Writing region close event to WAL at 1732376473096Closed at 1732376473096 2024-11-23T15:41:13,098 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T15:41:13,098 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732376473098"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732376473098"}]},"ts":"1732376473098"} 2024-11-23T15:41:13,100 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T15:41:13,101 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T15:41:13,101 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376473101"}]},"ts":"1732376473101"} 2024-11-23T15:41:13,103 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T15:41:13,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2607dde3b91c49826fdae09a26d7b59, ASSIGN}] 2024-11-23T15:41:13,104 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2607dde3b91c49826fdae09a26d7b59, ASSIGN 2024-11-23T15:41:13,104 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2607dde3b91c49826fdae09a26d7b59, ASSIGN; state=OFFLINE, location=b712f9af2c12,46589,1732376409194; forceNewPlan=false, retain=false 2024-11-23T15:41:13,255 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=d2607dde3b91c49826fdae09a26d7b59, regionState=OPENING, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:13,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2607dde3b91c49826fdae09a26d7b59, ASSIGN because future has completed 2024-11-23T15:41:13,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194}] 2024-11-23T15:41:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-23T15:41:13,425 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:13,425 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7752): Opening region: {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:41:13,425 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:13,425 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:41:13,425 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7794): checking encryption for d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:13,425 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7797): checking classloading for d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:13,427 INFO [StoreOpener-d2607dde3b91c49826fdae09a26d7b59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:13,428 INFO [StoreOpener-d2607dde3b91c49826fdae09a26d7b59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2607dde3b91c49826fdae09a26d7b59 columnFamilyName A 2024-11-23T15:41:13,428 DEBUG [StoreOpener-d2607dde3b91c49826fdae09a26d7b59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:13,429 INFO [StoreOpener-d2607dde3b91c49826fdae09a26d7b59-1 {}] regionserver.HStore(327): Store=d2607dde3b91c49826fdae09a26d7b59/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:41:13,429 INFO [StoreOpener-d2607dde3b91c49826fdae09a26d7b59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:13,430 INFO [StoreOpener-d2607dde3b91c49826fdae09a26d7b59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2607dde3b91c49826fdae09a26d7b59 columnFamilyName B 2024-11-23T15:41:13,430 DEBUG [StoreOpener-d2607dde3b91c49826fdae09a26d7b59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:13,430 INFO [StoreOpener-d2607dde3b91c49826fdae09a26d7b59-1 {}] regionserver.HStore(327): Store=d2607dde3b91c49826fdae09a26d7b59/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:41:13,430 INFO [StoreOpener-d2607dde3b91c49826fdae09a26d7b59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:13,431 INFO [StoreOpener-d2607dde3b91c49826fdae09a26d7b59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2607dde3b91c49826fdae09a26d7b59 columnFamilyName C 2024-11-23T15:41:13,431 DEBUG [StoreOpener-d2607dde3b91c49826fdae09a26d7b59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:13,432 INFO [StoreOpener-d2607dde3b91c49826fdae09a26d7b59-1 {}] regionserver.HStore(327): Store=d2607dde3b91c49826fdae09a26d7b59/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:41:13,432 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1038): replaying wal for d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:13,433 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:13,433 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:13,434 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1048): stopping wal replay for d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:13,434 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1060): Cleaning up temporary data for d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:13,434 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T15:41:13,435 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1093): writing seq id for d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:13,437 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:41:13,438 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1114): Opened d2607dde3b91c49826fdae09a26d7b59; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63821528, jitterRate=-0.04898512363433838}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T15:41:13,438 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:13,438 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1006): Region open journal for d2607dde3b91c49826fdae09a26d7b59: Running coprocessor pre-open hook at 1732376473425Writing region info on filesystem at 1732376473425Initializing all the Stores at 1732376473426 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376473426Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376473427 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376473427Cleaning up temporary data from old regions at 1732376473434 (+7 ms)Running coprocessor post-open hooks at 1732376473438 (+4 ms)Region opened successfully at 1732376473438 2024-11-23T15:41:13,439 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., pid=69, masterSystemTime=1732376473414 2024-11-23T15:41:13,441 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:13,441 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:13,441 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=d2607dde3b91c49826fdae09a26d7b59, regionState=OPEN, openSeqNum=2, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:13,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 because future has completed 2024-11-23T15:41:13,444 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-23T15:41:13,444 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 in 183 msec 2024-11-23T15:41:13,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=67 2024-11-23T15:41:13,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=67, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2607dde3b91c49826fdae09a26d7b59, ASSIGN in 342 msec 2024-11-23T15:41:13,447 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T15:41:13,447 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376473447"}]},"ts":"1732376473447"} 2024-11-23T15:41:13,448 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T15:41:13,449 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T15:41:13,450 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 778 msec 2024-11-23T15:41:13,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-23T15:41:13,816 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:13,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,819 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:41:13,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:41:13,822 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:41:13,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:41:13,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:41:13,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@755b5c49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:41:13,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:41:13,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:13,827 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:41:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@456abfc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,827 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:41:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:41:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:41:13,830 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:41:13,831 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T15:41:13,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is b712f9af2c12,42351,1732376408377 2024-11-23T15:41:13,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5f7db9b8 2024-11-23T15:41:13,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T15:41:13,832 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T15:41:13,834 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,834 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:41:13,834 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:41:13,835 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:41:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:41:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:41:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c3f22a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:41:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:41:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:13,836 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:41:13,836 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b3f6f80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,837 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:41:13,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:41:13,838 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:41:13,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:41:13,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:41:13,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21b732cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:41:13,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:41:13,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:13,839 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:41:13,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25d85752, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,841 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:41:13,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:41:13,841 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:41:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:41:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:41:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c92d725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:41:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:41:13,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:13,843 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:41:13,843 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec189c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,844 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:41:13,844 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:41:13,845 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:41:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:41:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:41:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40808aa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:41:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:41:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:13,846 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:41:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b62ceb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,848 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:41:13,848 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:41:13,849 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:41:13,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:41:13,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:41:13,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@212bc44d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:41:13,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:41:13,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:13,850 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:41:13,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e3581b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,852 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:41:13,852 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:41:13,852 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:41:13,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:41:13,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:41:13,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@eadb180, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:41:13,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:41:13,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:13,854 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:41:13,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c5400d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,855 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:41:13,855 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:41:13,856 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:41:13,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:41:13,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:41:13,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67518330, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:41:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:41:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:13,857 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:41:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ceb430, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,859 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:41:13,859 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:41:13,860 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:41:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:41:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:41:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69c25519, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:41:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:41:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:13,861 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:41:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa07b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,862 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:41:13,863 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:41:13,863 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:41:13,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:41:13,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:41:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70bc42b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:41:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:41:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:13,865 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:41:13,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28650e78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,866 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,866 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:41:13,866 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:41:13,866 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:41:13,866 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:41:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:41:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519d7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:41:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:41:13,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:13,867 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34898, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:41:13,868 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@432884ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:13,869 DEBUG [Thread-1134 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:41:13,869 DEBUG [Thread-1127 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:41:13,869 DEBUG [Thread-1140 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:41:13,869 DEBUG [Thread-1123 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:41:13,869 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:41:13,869 DEBUG [Thread-1136 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:41:13,870 DEBUG [Thread-1142 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:41:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:41:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:41:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:41:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:41:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:41:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:41:13,870 DEBUG [Thread-1125 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:41:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:41:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:41:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:41:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:41:13,871 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:41:13,871 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:41:13,871 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:41:13,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-23T15:41:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:41:13,872 DEBUG [Thread-1131 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:41:13,872 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:41:13,872 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:41:13,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:41:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:41:13,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-23T15:41:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:41:13,873 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:41:13,873 DEBUG [Thread-1129 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:41:13,874 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:41:13,874 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:41:13,874 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:41:13,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:41:13,874 DEBUG [Thread-1138 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:41:13,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:41:13,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:41:13,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:41:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:41:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:41:13,875 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:41:13,876 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:41:13,876 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:41:13,876 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:41:13,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:41:13,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:41:13,877 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:41:13,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:41:13,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:41:13,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:41:13,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:41:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:41:13,879 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:41:13,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:41:13,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:13,885 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:41:13,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T15:41:13,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376533910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:13,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:13,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:13,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:13,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376533911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:13,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/638edb96809d4f9fa851a1be1b0b5b72 is 50, key is test_row_0/A:col10/1732376473885/Put/seqid=0 2024-11-23T15:41:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:13,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:13,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376533912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:13,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:13,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376533914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:13,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:13,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376533916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:13,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:13,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:13,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:13,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:13,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:13,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:13,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742066_1242 (size=16681) 2024-11-23T15:41:13,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/638edb96809d4f9fa851a1be1b0b5b72 2024-11-23T15:41:13,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/4686e0b4d0ab480db7fa7dbec61a48ae is 50, key is test_row_0/B:col10/1732376473885/Put/seqid=0 2024-11-23T15:41:13,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742067_1243 (size=12001) 2024-11-23T15:41:13,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-23T15:41:14,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376534024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376534025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376534025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376534026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376534026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,031 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-23T15:41:14,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:14,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:14,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:14,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:14,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:14,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:14,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-23T15:41:14,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:14,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:14,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:14,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:14,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:14,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-23T15:41:14,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376534234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376534234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376534234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376534234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376534234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-23T15:41:14,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:14,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:14,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:14,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:14,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:14,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:14,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/4686e0b4d0ab480db7fa7dbec61a48ae 2024-11-23T15:41:14,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/acf92422bfc249d7b829e0b1cc384619 is 50, key is test_row_0/C:col10/1732376473885/Put/seqid=0 2024-11-23T15:41:14,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742068_1244 (size=12001) 2024-11-23T15:41:14,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/acf92422bfc249d7b829e0b1cc384619 2024-11-23T15:41:14,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/638edb96809d4f9fa851a1be1b0b5b72 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/638edb96809d4f9fa851a1be1b0b5b72 2024-11-23T15:41:14,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/638edb96809d4f9fa851a1be1b0b5b72, entries=250, sequenceid=15, filesize=16.3 K 2024-11-23T15:41:14,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/4686e0b4d0ab480db7fa7dbec61a48ae as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/4686e0b4d0ab480db7fa7dbec61a48ae 2024-11-23T15:41:14,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/4686e0b4d0ab480db7fa7dbec61a48ae, entries=150, sequenceid=15, filesize=11.7 K 2024-11-23T15:41:14,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/acf92422bfc249d7b829e0b1cc384619 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/acf92422bfc249d7b829e0b1cc384619 2024-11-23T15:41:14,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/acf92422bfc249d7b829e0b1cc384619, entries=150, sequenceid=15, filesize=11.7 K 2024-11-23T15:41:14,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for d2607dde3b91c49826fdae09a26d7b59 in 557ms, sequenceid=15, compaction requested=false 2024-11-23T15:41:14,442 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-23T15:41:14,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:14,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-23T15:41:14,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:14,489 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T15:41:14,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/2bfebe086d81466eb50ef42306993d45 is 50, key is test_row_0/A:col10/1732376473906/Put/seqid=0 2024-11-23T15:41:14,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742069_1245 (size=12001) 2024-11-23T15:41:14,500 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/2bfebe086d81466eb50ef42306993d45 2024-11-23T15:41:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-23T15:41:14,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/8e54b1561d67472c9287b6ecc1c22c20 is 50, key is test_row_0/B:col10/1732376473906/Put/seqid=0 2024-11-23T15:41:14,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742070_1246 (size=12001) 2024-11-23T15:41:14,517 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/8e54b1561d67472c9287b6ecc1c22c20 2024-11-23T15:41:14,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/39e4069f94114d238b039f912deb3945 is 50, key is test_row_0/C:col10/1732376473906/Put/seqid=0 2024-11-23T15:41:14,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742071_1247 (size=12001) 2024-11-23T15:41:14,530 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/39e4069f94114d238b039f912deb3945 2024-11-23T15:41:14,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/2bfebe086d81466eb50ef42306993d45 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/2bfebe086d81466eb50ef42306993d45 2024-11-23T15:41:14,538 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/2bfebe086d81466eb50ef42306993d45, entries=150, sequenceid=38, filesize=11.7 K 2024-11-23T15:41:14,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/8e54b1561d67472c9287b6ecc1c22c20 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/8e54b1561d67472c9287b6ecc1c22c20 2024-11-23T15:41:14,546 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/8e54b1561d67472c9287b6ecc1c22c20, entries=150, sequenceid=38, filesize=11.7 K 2024-11-23T15:41:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:14,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:14,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/39e4069f94114d238b039f912deb3945 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/39e4069f94114d238b039f912deb3945 2024-11-23T15:41:14,555 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/39e4069f94114d238b039f912deb3945, entries=150, sequenceid=38, filesize=11.7 K 2024-11-23T15:41:14,556 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d2607dde3b91c49826fdae09a26d7b59 in 67ms, sequenceid=38, compaction requested=false 2024-11-23T15:41:14,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:14,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:14,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-23T15:41:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-23T15:41:14,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:14,559 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:41:14,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T15:41:14,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-23T15:41:14,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 684 msec 2024-11-23T15:41:14,563 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 691 msec 2024-11-23T15:41:14,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/ced9086b11c74206bc07d4755d841e8e is 50, key is test_row_0/A:col10/1732376474552/Put/seqid=0 2024-11-23T15:41:14,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742072_1248 (size=12001) 2024-11-23T15:41:14,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376534579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376534582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376534582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376534583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376534584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376534694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376534694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376534694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376534694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376534694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376534904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376534904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376534904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376534904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:14,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376534904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:14,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:14,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:14,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/ced9086b11c74206bc07d4755d841e8e 2024-11-23T15:41:14,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/6d945cd1d47046b59836c71759a50357 is 50, key is test_row_0/B:col10/1732376474552/Put/seqid=0 2024-11-23T15:41:14,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742073_1249 (size=12001) 2024-11-23T15:41:15,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-23T15:41:15,015 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:15,015 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:41:15,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-23T15:41:15,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T15:41:15,017 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:41:15,017 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:41:15,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:41:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T15:41:15,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-23T15:41:15,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:15,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:15,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:15,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:15,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:15,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376535214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:15,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376535214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376535214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:15,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376535214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376535214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:15,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-23T15:41:15,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:15,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:15,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:15,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:15,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T15:41:15,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/6d945cd1d47046b59836c71759a50357 2024-11-23T15:41:15,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/6c55da66c480454287c5f44f554ab712 is 50, key is test_row_0/C:col10/1732376474552/Put/seqid=0 2024-11-23T15:41:15,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742074_1250 (size=12001) 2024-11-23T15:41:15,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/6c55da66c480454287c5f44f554ab712 2024-11-23T15:41:15,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/ced9086b11c74206bc07d4755d841e8e as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/ced9086b11c74206bc07d4755d841e8e 2024-11-23T15:41:15,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/ced9086b11c74206bc07d4755d841e8e, entries=150, sequenceid=52, filesize=11.7 K 2024-11-23T15:41:15,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/6d945cd1d47046b59836c71759a50357 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/6d945cd1d47046b59836c71759a50357 2024-11-23T15:41:15,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/6d945cd1d47046b59836c71759a50357, entries=150, sequenceid=52, filesize=11.7 K 2024-11-23T15:41:15,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/6c55da66c480454287c5f44f554ab712 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/6c55da66c480454287c5f44f554ab712 2024-11-23T15:41:15,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/6c55da66c480454287c5f44f554ab712, entries=150, sequenceid=52, filesize=11.7 K 2024-11-23T15:41:15,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d2607dde3b91c49826fdae09a26d7b59 in 867ms, sequenceid=52, compaction requested=true 2024-11-23T15:41:15,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:15,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:41:15,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:15,426 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:15,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:41:15,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:15,426 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:15,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:41:15,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:15,427 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:15,427 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:15,427 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/A is initiating minor compaction (all files) 2024-11-23T15:41:15,427 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/B is initiating minor compaction (all files) 2024-11-23T15:41:15,427 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/B in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:15,427 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/A in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:15,427 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/4686e0b4d0ab480db7fa7dbec61a48ae, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/8e54b1561d67472c9287b6ecc1c22c20, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/6d945cd1d47046b59836c71759a50357] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=35.2 K 2024-11-23T15:41:15,427 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/638edb96809d4f9fa851a1be1b0b5b72, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/2bfebe086d81466eb50ef42306993d45, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/ced9086b11c74206bc07d4755d841e8e] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=39.7 K 2024-11-23T15:41:15,428 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 638edb96809d4f9fa851a1be1b0b5b72, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732376473883 2024-11-23T15:41:15,428 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 4686e0b4d0ab480db7fa7dbec61a48ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732376473883 2024-11-23T15:41:15,428 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e54b1561d67472c9287b6ecc1c22c20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732376473906 2024-11-23T15:41:15,428 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bfebe086d81466eb50ef42306993d45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732376473906 2024-11-23T15:41:15,429 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d945cd1d47046b59836c71759a50357, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732376474552 2024-11-23T15:41:15,429 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ced9086b11c74206bc07d4755d841e8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732376474552 2024-11-23T15:41:15,448 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#B#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:15,448 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/4527cb53b13b4612bd8a7c5c1a0eb5e9 is 50, key is test_row_0/B:col10/1732376474552/Put/seqid=0 2024-11-23T15:41:15,451 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#A#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:15,451 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/6d67efc44a1b4f89873d762032dde51e is 50, key is test_row_0/A:col10/1732376474552/Put/seqid=0 2024-11-23T15:41:15,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742075_1251 (size=12104) 2024-11-23T15:41:15,464 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/4527cb53b13b4612bd8a7c5c1a0eb5e9 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/4527cb53b13b4612bd8a7c5c1a0eb5e9 2024-11-23T15:41:15,469 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/B of d2607dde3b91c49826fdae09a26d7b59 into 4527cb53b13b4612bd8a7c5c1a0eb5e9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:15,469 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:15,469 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/B, priority=13, startTime=1732376475426; duration=0sec 2024-11-23T15:41:15,469 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:15,469 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:B 2024-11-23T15:41:15,469 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:15,471 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:15,471 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/C is initiating minor compaction (all files) 2024-11-23T15:41:15,471 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/C in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:15,471 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/acf92422bfc249d7b829e0b1cc384619, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/39e4069f94114d238b039f912deb3945, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/6c55da66c480454287c5f44f554ab712] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=35.2 K 2024-11-23T15:41:15,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742076_1252 (size=12104) 2024-11-23T15:41:15,472 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting acf92422bfc249d7b829e0b1cc384619, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732376473883 2024-11-23T15:41:15,473 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 39e4069f94114d238b039f912deb3945, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732376473906 2024-11-23T15:41:15,473 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c55da66c480454287c5f44f554ab712, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732376474552 2024-11-23T15:41:15,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-23T15:41:15,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:15,476 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-23T15:41:15,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/1744e2a29781470b8fd0988bc34518c9 is 50, key is test_row_0/A:col10/1732376474582/Put/seqid=0 2024-11-23T15:41:15,482 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#C#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:15,483 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/c739ffb6532c4162825f5d2ad23147ba is 50, key is test_row_0/C:col10/1732376474552/Put/seqid=0 2024-11-23T15:41:15,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742077_1253 (size=12001) 2024-11-23T15:41:15,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742078_1254 (size=12104) 2024-11-23T15:41:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T15:41:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:15,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:15,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:15,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:15,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376535735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:15,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376535735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:15,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:15,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376535736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:15,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:15,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:15,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:15,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376535739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:15,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:15,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:15,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376535739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:15,744 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:15,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:15,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:15,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376535854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:15,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376535854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376535854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:15,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376535854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:15,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376535854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:15,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:15,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:15,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:15,867 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T15:41:15,880 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/6d67efc44a1b4f89873d762032dde51e as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6d67efc44a1b4f89873d762032dde51e 2024-11-23T15:41:15,888 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/A of d2607dde3b91c49826fdae09a26d7b59 into 6d67efc44a1b4f89873d762032dde51e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:15,888 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:15,888 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/A, priority=13, startTime=1732376475426; duration=0sec 2024-11-23T15:41:15,889 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:15,889 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:A 2024-11-23T15:41:15,893 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/1744e2a29781470b8fd0988bc34518c9 2024-11-23T15:41:15,907 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/c739ffb6532c4162825f5d2ad23147ba as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/c739ffb6532c4162825f5d2ad23147ba 2024-11-23T15:41:15,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/6677d10f060645e1bb4681cf14166348 is 50, key is test_row_0/B:col10/1732376474582/Put/seqid=0 2024-11-23T15:41:15,918 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/C of d2607dde3b91c49826fdae09a26d7b59 into c739ffb6532c4162825f5d2ad23147ba(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:15,918 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:15,918 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/C, priority=13, startTime=1732376475426; duration=0sec 2024-11-23T15:41:15,918 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:15,919 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:C 2024-11-23T15:41:15,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742079_1255 (size=12001) 2024-11-23T15:41:15,926 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/6677d10f060645e1bb4681cf14166348 2024-11-23T15:41:15,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/6117a615fb2747129c014c974a396dbf is 50, key is test_row_0/C:col10/1732376474582/Put/seqid=0 2024-11-23T15:41:15,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742080_1256 (size=12001) 2024-11-23T15:41:15,947 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/6117a615fb2747129c014c974a396dbf 2024-11-23T15:41:15,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/1744e2a29781470b8fd0988bc34518c9 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/1744e2a29781470b8fd0988bc34518c9 2024-11-23T15:41:15,969 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/1744e2a29781470b8fd0988bc34518c9, entries=150, sequenceid=75, filesize=11.7 K 2024-11-23T15:41:15,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/6677d10f060645e1bb4681cf14166348 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/6677d10f060645e1bb4681cf14166348 2024-11-23T15:41:15,975 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/6677d10f060645e1bb4681cf14166348, entries=150, sequenceid=75, filesize=11.7 K 2024-11-23T15:41:15,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/6117a615fb2747129c014c974a396dbf as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/6117a615fb2747129c014c974a396dbf 2024-11-23T15:41:15,981 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/6117a615fb2747129c014c974a396dbf, entries=150, sequenceid=75, filesize=11.7 K 2024-11-23T15:41:15,981 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for d2607dde3b91c49826fdae09a26d7b59 in 505ms, sequenceid=75, compaction requested=false 2024-11-23T15:41:15,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:15,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:15,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-23T15:41:15,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-23T15:41:15,985 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-23T15:41:15,985 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 966 msec 2024-11-23T15:41:15,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 971 msec 2024-11-23T15:41:16,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:16,066 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:41:16,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-23T15:41:16,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/6e08e44f115d4723968963a20db33eb8 is 50, key is test_row_0/A:col10/1732376476066/Put/seqid=0 2024-11-23T15:41:16,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742081_1257 (size=12001) 2024-11-23T15:41:16,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:16,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:16,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376536080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:16,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376536081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:16,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:16,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376536086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:16,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:16,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:16,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376536086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:16,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:16,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:16,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376536086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:16,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:16,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T15:41:16,154 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:16,155 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:41:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-23T15:41:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T15:41:16,156 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:41:16,157 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:41:16,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:41:16,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:16,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376536195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:16,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376536195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376536195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376536195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376536195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:16,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:16,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T15:41:16,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T15:41:16,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:16,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:16,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:16,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:16,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:16,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376536404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:16,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:16,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376536404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:16,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:16,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:16,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376536404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:16,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:16,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376536404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:16,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:16,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:16,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376536404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:16,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:16,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T15:41:16,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:16,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:16,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:16,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:16,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:16,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T15:41:16,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/6e08e44f115d4723968963a20db33eb8 2024-11-23T15:41:16,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/3896dc1347ea4f9da0ea16998e4184c5 is 50, key is test_row_0/B:col10/1732376476066/Put/seqid=0 2024-11-23T15:41:16,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742082_1258 (size=12001) 2024-11-23T15:41:16,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T15:41:16,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:16,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:16,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:16,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:16,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:16,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:16,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:16,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376536714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:16,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:16,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376536714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:16,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376536714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376536714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:16,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376536714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:16,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:16,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:16,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T15:41:16,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:16,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:16,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:16,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:16,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:16,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:16,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T15:41:16,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/3896dc1347ea4f9da0ea16998e4184c5 2024-11-23T15:41:16,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T15:41:16,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:16,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:16,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:16,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:16,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:16,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:16,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/6bd128f800f74ee5825c61ba5267cd59 is 50, key is test_row_0/C:col10/1732376476066/Put/seqid=0 2024-11-23T15:41:16,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742083_1259 (size=12001) 2024-11-23T15:41:16,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/6bd128f800f74ee5825c61ba5267cd59 2024-11-23T15:41:16,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/6e08e44f115d4723968963a20db33eb8 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6e08e44f115d4723968963a20db33eb8 2024-11-23T15:41:16,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6e08e44f115d4723968963a20db33eb8, entries=150, sequenceid=95, filesize=11.7 K 2024-11-23T15:41:16,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/3896dc1347ea4f9da0ea16998e4184c5 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/3896dc1347ea4f9da0ea16998e4184c5 2024-11-23T15:41:16,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/3896dc1347ea4f9da0ea16998e4184c5, entries=150, sequenceid=95, filesize=11.7 K 2024-11-23T15:41:16,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/6bd128f800f74ee5825c61ba5267cd59 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/6bd128f800f74ee5825c61ba5267cd59 2024-11-23T15:41:16,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/6bd128f800f74ee5825c61ba5267cd59, entries=150, sequenceid=95, filesize=11.7 K 2024-11-23T15:41:16,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for d2607dde3b91c49826fdae09a26d7b59 in 930ms, sequenceid=95, compaction requested=true 2024-11-23T15:41:16,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:16,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:41:16,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:16,997 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:16,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:41:16,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:16,997 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:16,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:41:16,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:16,998 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:16,998 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/A is initiating minor compaction (all files) 2024-11-23T15:41:16,998 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/A in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:16,998 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6d67efc44a1b4f89873d762032dde51e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/1744e2a29781470b8fd0988bc34518c9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6e08e44f115d4723968963a20db33eb8] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=35.3 K 2024-11-23T15:41:17,000 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:17,000 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d67efc44a1b4f89873d762032dde51e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732376474552 2024-11-23T15:41:17,000 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/B is initiating minor compaction (all files) 2024-11-23T15:41:17,000 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/B in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:17,000 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/4527cb53b13b4612bd8a7c5c1a0eb5e9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/6677d10f060645e1bb4681cf14166348, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/3896dc1347ea4f9da0ea16998e4184c5] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=35.3 K 2024-11-23T15:41:17,000 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1744e2a29781470b8fd0988bc34518c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732376474572 2024-11-23T15:41:17,001 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 4527cb53b13b4612bd8a7c5c1a0eb5e9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732376474552 2024-11-23T15:41:17,001 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6677d10f060645e1bb4681cf14166348, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732376474572 2024-11-23T15:41:17,001 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e08e44f115d4723968963a20db33eb8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732376476065 2024-11-23T15:41:17,002 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 3896dc1347ea4f9da0ea16998e4184c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732376476065 2024-11-23T15:41:17,021 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#B#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:17,021 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#A#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:17,021 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/e91e1b8438004b62a195ebdc217ca89a is 50, key is test_row_0/A:col10/1732376476066/Put/seqid=0 2024-11-23T15:41:17,021 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/9a68f13fa1944297a89f1c0fb92f065d is 50, key is test_row_0/B:col10/1732376476066/Put/seqid=0 2024-11-23T15:41:17,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742084_1260 (size=12207) 2024-11-23T15:41:17,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742085_1261 (size=12207) 2024-11-23T15:41:17,051 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/9a68f13fa1944297a89f1c0fb92f065d as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/9a68f13fa1944297a89f1c0fb92f065d 2024-11-23T15:41:17,056 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/B of d2607dde3b91c49826fdae09a26d7b59 into 9a68f13fa1944297a89f1c0fb92f065d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:17,056 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:17,056 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/B, priority=13, startTime=1732376476997; duration=0sec 2024-11-23T15:41:17,056 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:17,056 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:B 2024-11-23T15:41:17,056 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:17,058 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:17,058 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/C is initiating minor compaction (all files) 2024-11-23T15:41:17,058 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/C in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:17,058 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/c739ffb6532c4162825f5d2ad23147ba, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/6117a615fb2747129c014c974a396dbf, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/6bd128f800f74ee5825c61ba5267cd59] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=35.3 K 2024-11-23T15:41:17,059 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting c739ffb6532c4162825f5d2ad23147ba, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732376474552 2024-11-23T15:41:17,059 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6117a615fb2747129c014c974a396dbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732376474572 2024-11-23T15:41:17,059 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bd128f800f74ee5825c61ba5267cd59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732376476065 2024-11-23T15:41:17,067 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#C#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:17,068 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/230d133064584d3489b62add060d6579 is 50, key is test_row_0/C:col10/1732376476066/Put/seqid=0 2024-11-23T15:41:17,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742086_1262 (size=12207) 2024-11-23T15:41:17,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T15:41:17,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:17,076 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-23T15:41:17,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/52c4ba5afb9c42ca81bd1a4ada310b89 is 50, key is test_row_0/A:col10/1732376476080/Put/seqid=0 2024-11-23T15:41:17,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742087_1263 (size=12001) 2024-11-23T15:41:17,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:17,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:17,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:17,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376537243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:17,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:17,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376537246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:17,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:17,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376537247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:17,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376537247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:17,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:17,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376537247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:17,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:17,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T15:41:17,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:17,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376537354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:17,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:17,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376537354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:17,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376537364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:17,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:17,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376537364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:17,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:17,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:17,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376537364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:17,454 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/e91e1b8438004b62a195ebdc217ca89a as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/e91e1b8438004b62a195ebdc217ca89a 2024-11-23T15:41:17,459 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/A of d2607dde3b91c49826fdae09a26d7b59 into e91e1b8438004b62a195ebdc217ca89a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:17,459 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:17,459 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/A, priority=13, startTime=1732376476997; duration=0sec 2024-11-23T15:41:17,459 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:17,459 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:A 2024-11-23T15:41:17,477 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/230d133064584d3489b62add060d6579 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/230d133064584d3489b62add060d6579 2024-11-23T15:41:17,482 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/C of d2607dde3b91c49826fdae09a26d7b59 into 230d133064584d3489b62add060d6579(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:17,482 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:17,482 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/C, priority=13, startTime=1732376476997; duration=0sec 2024-11-23T15:41:17,482 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:17,482 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:C 2024-11-23T15:41:17,512 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/52c4ba5afb9c42ca81bd1a4ada310b89 2024-11-23T15:41:17,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/9f1eb00a8c754d2c97e0e9c8b36b2272 is 50, key is test_row_0/B:col10/1732376476080/Put/seqid=0 2024-11-23T15:41:17,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742088_1264 (size=12001) 2024-11-23T15:41:17,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:17,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376537564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:17,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376537564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:17,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:17,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376537574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:17,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:17,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376537574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:17,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:17,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:17,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:17,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376537574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:17,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:17,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376537874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376537874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:17,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376537884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:17,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376537884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:17,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376537884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:17,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:17,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:17,923 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/9f1eb00a8c754d2c97e0e9c8b36b2272 2024-11-23T15:41:17,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/0a545a3024004e2a93077eb565f88db1 is 50, key is test_row_0/C:col10/1732376476080/Put/seqid=0 2024-11-23T15:41:17,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742089_1265 (size=12001) 2024-11-23T15:41:17,948 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/0a545a3024004e2a93077eb565f88db1 2024-11-23T15:41:17,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/52c4ba5afb9c42ca81bd1a4ada310b89 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/52c4ba5afb9c42ca81bd1a4ada310b89 2024-11-23T15:41:17,963 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/52c4ba5afb9c42ca81bd1a4ada310b89, entries=150, sequenceid=115, filesize=11.7 K 2024-11-23T15:41:17,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/9f1eb00a8c754d2c97e0e9c8b36b2272 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/9f1eb00a8c754d2c97e0e9c8b36b2272 2024-11-23T15:41:17,972 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/9f1eb00a8c754d2c97e0e9c8b36b2272, entries=150, sequenceid=115, filesize=11.7 K 2024-11-23T15:41:17,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/0a545a3024004e2a93077eb565f88db1 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/0a545a3024004e2a93077eb565f88db1 2024-11-23T15:41:17,980 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/0a545a3024004e2a93077eb565f88db1, entries=150, sequenceid=115, filesize=11.7 K 2024-11-23T15:41:17,981 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for d2607dde3b91c49826fdae09a26d7b59 in 905ms, sequenceid=115, compaction requested=false 2024-11-23T15:41:17,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:17,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:17,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-23T15:41:17,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-23T15:41:17,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-23T15:41:17,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8260 sec 2024-11-23T15:41:17,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.8320 sec 2024-11-23T15:41:18,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T15:41:18,304 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:18,305 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:41:18,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-23T15:41:18,309 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:41:18,310 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:41:18,310 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:41:18,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T15:41:18,387 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:41:18,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-23T15:41:18,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:18,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/ab3c6a691117411fa9302f246f02e9f8 is 50, key is test_row_0/A:col10/1732376477245/Put/seqid=0 2024-11-23T15:41:18,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T15:41:18,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:18,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376538411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:18,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:18,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376538414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:18,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:18,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376538416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:18,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:18,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376538419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:18,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:18,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:18,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376538419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:18,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:18,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742090_1266 (size=12151) 2024-11-23T15:41:18,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/ab3c6a691117411fa9302f246f02e9f8 2024-11-23T15:41:18,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-23T15:41:18,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:18,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:18,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:18,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:18,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:18,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/36ed39207174475985821bd619051ea6 is 50, key is test_row_0/B:col10/1732376477245/Put/seqid=0 2024-11-23T15:41:18,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742091_1267 (size=12151) 2024-11-23T15:41:18,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/36ed39207174475985821bd619051ea6 2024-11-23T15:41:18,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/f806cea6d94148129b384f99215d3d7d is 50, key is test_row_0/C:col10/1732376477245/Put/seqid=0 2024-11-23T15:41:18,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742092_1268 (size=12151) 2024-11-23T15:41:18,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376538524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:18,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376538534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376538534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:18,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:18,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376538534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:18,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376538534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:18,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:18,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:18,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-23T15:41:18,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:18,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:18,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:18,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:18,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:18,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T15:41:18,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376538734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:18,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:18,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376538744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:18,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:18,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376538744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:18,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,749 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:18,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376538744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:18,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376538744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:18,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:18,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:18,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:18,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-23T15:41:18,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:18,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:18,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:18,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:18,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:18,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-23T15:41:18,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:18,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:18,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:18,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:18,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:18,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/f806cea6d94148129b384f99215d3d7d 2024-11-23T15:41:18,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T15:41:18,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/ab3c6a691117411fa9302f246f02e9f8 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/ab3c6a691117411fa9302f246f02e9f8 2024-11-23T15:41:18,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/ab3c6a691117411fa9302f246f02e9f8, entries=150, sequenceid=136, filesize=11.9 K 2024-11-23T15:41:18,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/36ed39207174475985821bd619051ea6 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/36ed39207174475985821bd619051ea6 2024-11-23T15:41:18,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/36ed39207174475985821bd619051ea6, entries=150, sequenceid=136, filesize=11.9 K 2024-11-23T15:41:18,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/f806cea6d94148129b384f99215d3d7d as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/f806cea6d94148129b384f99215d3d7d 2024-11-23T15:41:18,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/f806cea6d94148129b384f99215d3d7d, entries=150, sequenceid=136, filesize=11.9 K 2024-11-23T15:41:18,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for d2607dde3b91c49826fdae09a26d7b59 in 572ms, sequenceid=136, compaction requested=true 2024-11-23T15:41:18,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:18,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:41:18,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:18,959 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:18,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:41:18,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:18,959 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:18,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:41:18,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:18,960 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:18,960 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/A is initiating minor compaction (all files) 2024-11-23T15:41:18,960 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/A in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:18,961 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/e91e1b8438004b62a195ebdc217ca89a, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/52c4ba5afb9c42ca81bd1a4ada310b89, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/ab3c6a691117411fa9302f246f02e9f8] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=35.5 K 2024-11-23T15:41:18,961 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:18,961 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/B is initiating minor compaction (all files) 2024-11-23T15:41:18,961 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/B in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:18,961 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting e91e1b8438004b62a195ebdc217ca89a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732376476065 2024-11-23T15:41:18,961 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/9a68f13fa1944297a89f1c0fb92f065d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/9f1eb00a8c754d2c97e0e9c8b36b2272, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/36ed39207174475985821bd619051ea6] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=35.5 K 2024-11-23T15:41:18,961 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a68f13fa1944297a89f1c0fb92f065d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732376476065 2024-11-23T15:41:18,961 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52c4ba5afb9c42ca81bd1a4ada310b89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732376476080 2024-11-23T15:41:18,961 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f1eb00a8c754d2c97e0e9c8b36b2272, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732376476080 2024-11-23T15:41:18,962 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 36ed39207174475985821bd619051ea6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732376477245 2024-11-23T15:41:18,963 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab3c6a691117411fa9302f246f02e9f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732376477245 2024-11-23T15:41:18,988 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#B#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:18,988 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/1199f9b293de47caa24e8d67ccc061f5 is 50, key is test_row_0/B:col10/1732376477245/Put/seqid=0 2024-11-23T15:41:18,997 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#A#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:18,997 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/901a58b0bc9d47a691a3477c84ba9bcb is 50, key is test_row_0/A:col10/1732376477245/Put/seqid=0 2024-11-23T15:41:19,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742093_1269 (size=12459) 2024-11-23T15:41:19,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742094_1270 (size=12459) 2024-11-23T15:41:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:19,050 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:41:19,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-23T15:41:19,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/9b17080f80534e709009559586e6b751 is 50, key is test_row_0/A:col10/1732376478418/Put/seqid=0 2024-11-23T15:41:19,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-23T15:41:19,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:19,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:19,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:19,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:19,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:19,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:19,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376539079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376539082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376539082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376539084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376539084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742095_1271 (size=14541) 2024-11-23T15:41:19,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/9b17080f80534e709009559586e6b751 2024-11-23T15:41:19,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/a61089bd13da4910ae89088cb896c468 is 50, key is test_row_0/B:col10/1732376478418/Put/seqid=0 2024-11-23T15:41:19,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742096_1272 (size=12151) 2024-11-23T15:41:19,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/a61089bd13da4910ae89088cb896c468 2024-11-23T15:41:19,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/665f0d06503543bb9d5b3875e9741f90 is 50, key is test_row_0/C:col10/1732376478418/Put/seqid=0 2024-11-23T15:41:19,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376539194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376539194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376539195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376539195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376539195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742097_1273 (size=12151) 2024-11-23T15:41:19,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/665f0d06503543bb9d5b3875e9741f90 2024-11-23T15:41:19,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/9b17080f80534e709009559586e6b751 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/9b17080f80534e709009559586e6b751 2024-11-23T15:41:19,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-23T15:41:19,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:19,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:19,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:19,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:19,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:19,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/9b17080f80534e709009559586e6b751, entries=200, sequenceid=155, filesize=14.2 K 2024-11-23T15:41:19,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/a61089bd13da4910ae89088cb896c468 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/a61089bd13da4910ae89088cb896c468 2024-11-23T15:41:19,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/a61089bd13da4910ae89088cb896c468, entries=150, sequenceid=155, filesize=11.9 K 2024-11-23T15:41:19,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/665f0d06503543bb9d5b3875e9741f90 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/665f0d06503543bb9d5b3875e9741f90 2024-11-23T15:41:19,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/665f0d06503543bb9d5b3875e9741f90, entries=150, sequenceid=155, filesize=11.9 K 2024-11-23T15:41:19,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for d2607dde3b91c49826fdae09a26d7b59 in 208ms, sequenceid=155, compaction requested=true 2024-11-23T15:41:19,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:19,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:A, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:41:19,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-23T15:41:19,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:B, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:41:19,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-23T15:41:19,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:41:19,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-23T15:41:19,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-23T15:41:19,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:19,385 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-23T15:41:19,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/b4456b929f324bc2b028e8a951521db7 is 50, key is test_row_0/A:col10/1732376479071/Put/seqid=0 2024-11-23T15:41:19,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:19,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:19,431 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/1199f9b293de47caa24e8d67ccc061f5 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/1199f9b293de47caa24e8d67ccc061f5 2024-11-23T15:41:19,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376539430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376539431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376539432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376539433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376539435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T15:41:19,455 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/B of d2607dde3b91c49826fdae09a26d7b59 into 1199f9b293de47caa24e8d67ccc061f5(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:19,455 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:19,455 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/B, priority=13, startTime=1732376478959; duration=0sec 2024-11-23T15:41:19,456 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-23T15:41:19,456 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:B 2024-11-23T15:41:19,456 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-23T15:41:19,456 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-23T15:41:19,457 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-23T15:41:19,458 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/901a58b0bc9d47a691a3477c84ba9bcb as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/901a58b0bc9d47a691a3477c84ba9bcb 2024-11-23T15:41:19,459 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. because compaction request was cancelled 2024-11-23T15:41:19,459 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:A 2024-11-23T15:41:19,459 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:41:19,461 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:41:19,461 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/C is initiating minor compaction (all files) 2024-11-23T15:41:19,461 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/C in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:19,462 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/230d133064584d3489b62add060d6579, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/0a545a3024004e2a93077eb565f88db1, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/f806cea6d94148129b384f99215d3d7d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/665f0d06503543bb9d5b3875e9741f90] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=47.4 K 2024-11-23T15:41:19,462 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 230d133064584d3489b62add060d6579, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732376476065 2024-11-23T15:41:19,463 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a545a3024004e2a93077eb565f88db1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732376476080 2024-11-23T15:41:19,463 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/A of d2607dde3b91c49826fdae09a26d7b59 into 901a58b0bc9d47a691a3477c84ba9bcb(size=12.2 K), total size for store is 26.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:19,464 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:19,464 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/A, priority=13, startTime=1732376478959; duration=0sec 2024-11-23T15:41:19,464 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-23T15:41:19,464 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:A 2024-11-23T15:41:19,464 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting f806cea6d94148129b384f99215d3d7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732376477245 2024-11-23T15:41:19,464 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:B 2024-11-23T15:41:19,464 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-23T15:41:19,464 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-23T15:41:19,464 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-23T15:41:19,464 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. because compaction request was cancelled 2024-11-23T15:41:19,464 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:C 2024-11-23T15:41:19,464 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-23T15:41:19,464 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 665f0d06503543bb9d5b3875e9741f90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732376478407 2024-11-23T15:41:19,465 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-23T15:41:19,465 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-23T15:41:19,465 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. because compaction request was cancelled 2024-11-23T15:41:19,465 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:B 2024-11-23T15:41:19,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742098_1274 (size=12151) 2024-11-23T15:41:19,476 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/b4456b929f324bc2b028e8a951521db7 2024-11-23T15:41:19,482 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#C#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:19,482 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/0e1f5284df964499b0dfad3c3c7c9cfd is 50, key is test_row_0/C:col10/1732376478418/Put/seqid=0 2024-11-23T15:41:19,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/d6a26c78c00c4e44ab7060937d990971 is 50, key is test_row_0/B:col10/1732376479071/Put/seqid=0 2024-11-23T15:41:19,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742099_1275 (size=12493) 2024-11-23T15:41:19,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376539544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376539544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376539544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376539545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376539545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742100_1276 (size=12151) 2024-11-23T15:41:19,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376539755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376539755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376539755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376539755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376539755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:19,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:19,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:19,935 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/0e1f5284df964499b0dfad3c3c7c9cfd as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/0e1f5284df964499b0dfad3c3c7c9cfd 2024-11-23T15:41:19,942 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/C of d2607dde3b91c49826fdae09a26d7b59 into 0e1f5284df964499b0dfad3c3c7c9cfd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:19,942 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:19,942 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/C, priority=12, startTime=1732376479258; duration=0sec 2024-11-23T15:41:19,942 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:19,942 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:C 2024-11-23T15:41:19,952 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/d6a26c78c00c4e44ab7060937d990971 2024-11-23T15:41:19,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/4ae1dd7445b84618885ff4882f73b37b is 50, key is test_row_0/C:col10/1732376479071/Put/seqid=0 2024-11-23T15:41:19,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742101_1277 (size=12151) 2024-11-23T15:41:20,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:20,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376540064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:20,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:20,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376540064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:20,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:20,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376540065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:20,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:20,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376540065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376540065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:20,373 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/4ae1dd7445b84618885ff4882f73b37b 2024-11-23T15:41:20,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/b4456b929f324bc2b028e8a951521db7 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/b4456b929f324bc2b028e8a951521db7 2024-11-23T15:41:20,390 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/b4456b929f324bc2b028e8a951521db7, entries=150, sequenceid=173, filesize=11.9 K 2024-11-23T15:41:20,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/d6a26c78c00c4e44ab7060937d990971 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/d6a26c78c00c4e44ab7060937d990971 2024-11-23T15:41:20,398 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/d6a26c78c00c4e44ab7060937d990971, entries=150, sequenceid=173, filesize=11.9 K 2024-11-23T15:41:20,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/4ae1dd7445b84618885ff4882f73b37b as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/4ae1dd7445b84618885ff4882f73b37b 2024-11-23T15:41:20,405 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/4ae1dd7445b84618885ff4882f73b37b, entries=150, sequenceid=173, filesize=11.9 K 2024-11-23T15:41:20,406 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for d2607dde3b91c49826fdae09a26d7b59 in 1021ms, sequenceid=173, compaction requested=true 2024-11-23T15:41:20,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:20,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:20,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-23T15:41:20,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-23T15:41:20,410 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-23T15:41:20,410 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0980 sec 2024-11-23T15:41:20,414 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.1070 sec 2024-11-23T15:41:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T15:41:20,454 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:20,455 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:41:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-23T15:41:20,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-23T15:41:20,458 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:41:20,459 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:41:20,459 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:41:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-23T15:41:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:20,577 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:41:20,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-23T15:41:20,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/657154ca44cc49528f1749fbcc898c02 is 50, key is test_row_0/A:col10/1732376479433/Put/seqid=0 2024-11-23T15:41:20,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742102_1278 (size=12151) 2024-11-23T15:41:20,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/657154ca44cc49528f1749fbcc898c02 2024-11-23T15:41:20,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:20,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376540600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376540600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:20,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:20,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376540600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376540601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:20,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:20,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376540607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:20,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:20,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T15:41:20,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:20,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:20,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:20,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:20,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:20,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:20,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/7675c227ee95421d88030a96e0e2344b is 50, key is test_row_0/B:col10/1732376479433/Put/seqid=0 2024-11-23T15:41:20,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742103_1279 (size=12151) 2024-11-23T15:41:20,645 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/7675c227ee95421d88030a96e0e2344b 2024-11-23T15:41:20,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/aafe0c70717844f0a64a3e3ca5516a17 is 50, key is test_row_0/C:col10/1732376479433/Put/seqid=0 2024-11-23T15:41:20,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742104_1280 (size=12151) 2024-11-23T15:41:20,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/aafe0c70717844f0a64a3e3ca5516a17 2024-11-23T15:41:20,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/657154ca44cc49528f1749fbcc898c02 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/657154ca44cc49528f1749fbcc898c02 2024-11-23T15:41:20,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/657154ca44cc49528f1749fbcc898c02, entries=150, sequenceid=195, filesize=11.9 K 2024-11-23T15:41:20,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/7675c227ee95421d88030a96e0e2344b as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/7675c227ee95421d88030a96e0e2344b 2024-11-23T15:41:20,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/7675c227ee95421d88030a96e0e2344b, entries=150, sequenceid=195, filesize=11.9 K 2024-11-23T15:41:20,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/aafe0c70717844f0a64a3e3ca5516a17 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/aafe0c70717844f0a64a3e3ca5516a17 2024-11-23T15:41:20,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/aafe0c70717844f0a64a3e3ca5516a17, entries=150, sequenceid=195, filesize=11.9 K 2024-11-23T15:41:20,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for d2607dde3b91c49826fdae09a26d7b59 in 131ms, sequenceid=195, compaction requested=true 2024-11-23T15:41:20,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:20,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:41:20,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:20,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:41:20,708 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:41:20,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:20,708 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:41:20,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:41:20,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:20,710 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:41:20,710 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/B is initiating minor compaction (all files) 2024-11-23T15:41:20,710 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/B in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:20,710 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/1199f9b293de47caa24e8d67ccc061f5, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/a61089bd13da4910ae89088cb896c468, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/d6a26c78c00c4e44ab7060937d990971, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/7675c227ee95421d88030a96e0e2344b] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=47.8 K 2024-11-23T15:41:20,710 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51302 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:41:20,710 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/A is initiating minor compaction (all files) 2024-11-23T15:41:20,710 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/A in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:20,710 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/901a58b0bc9d47a691a3477c84ba9bcb, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/9b17080f80534e709009559586e6b751, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/b4456b929f324bc2b028e8a951521db7, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/657154ca44cc49528f1749fbcc898c02] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=50.1 K 2024-11-23T15:41:20,711 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 901a58b0bc9d47a691a3477c84ba9bcb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732376477245 2024-11-23T15:41:20,711 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b17080f80534e709009559586e6b751, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732376478407 2024-11-23T15:41:20,711 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 1199f9b293de47caa24e8d67ccc061f5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732376477245 2024-11-23T15:41:20,711 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4456b929f324bc2b028e8a951521db7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732376479071 2024-11-23T15:41:20,711 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting a61089bd13da4910ae89088cb896c468, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732376478407 2024-11-23T15:41:20,711 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 657154ca44cc49528f1749fbcc898c02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732376479428 2024-11-23T15:41:20,712 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting d6a26c78c00c4e44ab7060937d990971, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732376479071 2024-11-23T15:41:20,712 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 7675c227ee95421d88030a96e0e2344b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732376479428 2024-11-23T15:41:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:20,718 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:41:20,718 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-23T15:41:20,723 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#A#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:20,725 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#B#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:20,725 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/58ae63165a3e4e00b4e637038051e21f is 50, key is test_row_0/A:col10/1732376479433/Put/seqid=0 2024-11-23T15:41:20,725 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/a260bb45be1445e09f475d9f19951e57 is 50, key is test_row_0/B:col10/1732376479433/Put/seqid=0 2024-11-23T15:41:20,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/9bf8e924f6394291b77d97a87ededb98 is 50, key is test_row_0/A:col10/1732376480599/Put/seqid=0 2024-11-23T15:41:20,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742107_1283 (size=12151) 2024-11-23T15:41:20,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T15:41:20,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:20,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:20,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:20,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:20,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:20,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:20,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:20,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376540737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:20,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742105_1281 (size=12595) 2024-11-23T15:41:20,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376540766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:20,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:20,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376540766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:20,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376540766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:20,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742106_1282 (size=12595) 2024-11-23T15:41:20,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-23T15:41:20,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:20,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376540772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:20,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:20,779 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/58ae63165a3e4e00b4e637038051e21f as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/58ae63165a3e4e00b4e637038051e21f 2024-11-23T15:41:20,783 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/A of d2607dde3b91c49826fdae09a26d7b59 into 58ae63165a3e4e00b4e637038051e21f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:20,784 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:20,784 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/A, priority=12, startTime=1732376480708; duration=0sec 2024-11-23T15:41:20,784 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:20,784 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:A 2024-11-23T15:41:20,784 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:20,785 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:20,785 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/C is initiating minor compaction (all files) 2024-11-23T15:41:20,785 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/C in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:20,785 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/0e1f5284df964499b0dfad3c3c7c9cfd, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/4ae1dd7445b84618885ff4882f73b37b, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/aafe0c70717844f0a64a3e3ca5516a17] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=35.9 K 2024-11-23T15:41:20,786 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e1f5284df964499b0dfad3c3c7c9cfd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732376478407 2024-11-23T15:41:20,786 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ae1dd7445b84618885ff4882f73b37b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732376479071 2024-11-23T15:41:20,786 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting aafe0c70717844f0a64a3e3ca5516a17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732376479428 2024-11-23T15:41:20,792 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#C#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:20,792 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/5124158e765e410ba999d1af8ad81bbe is 50, key is test_row_0/C:col10/1732376479433/Put/seqid=0 2024-11-23T15:41:20,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742108_1284 (size=12595) 2024-11-23T15:41:20,805 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/5124158e765e410ba999d1af8ad81bbe as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/5124158e765e410ba999d1af8ad81bbe 2024-11-23T15:41:20,810 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/C of d2607dde3b91c49826fdae09a26d7b59 into 5124158e765e410ba999d1af8ad81bbe(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:20,810 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:20,810 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/C, priority=13, startTime=1732376480708; duration=0sec 2024-11-23T15:41:20,811 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:20,811 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:C 2024-11-23T15:41:20,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:20,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376540874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376540874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:20,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:20,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376540884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:20,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:20,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:20,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:20,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376540884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:20,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376540884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:20,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:20,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:20,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:20,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T15:41:20,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:20,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:20,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:20,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:20,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:21,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T15:41:21,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:21,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:21,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:21,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:21,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:21,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-23T15:41:21,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376541084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:21,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:21,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376541084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:21,086 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:21,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:21,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:21,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376541094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:21,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:21,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376541095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:21,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:21,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:21,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376541095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:21,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:21,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:21,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/9bf8e924f6394291b77d97a87ededb98 2024-11-23T15:41:21,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/b45020ca5a734331a039fbca3b1aff04 is 50, key is test_row_0/B:col10/1732376480599/Put/seqid=0 2024-11-23T15:41:21,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742109_1285 (size=12151) 2024-11-23T15:41:21,190 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/a260bb45be1445e09f475d9f19951e57 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/a260bb45be1445e09f475d9f19951e57 2024-11-23T15:41:21,197 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/B of d2607dde3b91c49826fdae09a26d7b59 into a260bb45be1445e09f475d9f19951e57(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:21,198 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:21,198 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/B, priority=12, startTime=1732376480708; duration=0sec 2024-11-23T15:41:21,198 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:21,198 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:B 2024-11-23T15:41:21,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T15:41:21,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:21,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:21,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:21,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:21,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:21,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:21,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T15:41:21,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:21,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:21,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:21,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:21,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:21,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:21,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:21,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376541394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:21,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:21,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:21,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376541394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:21,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:21,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376541404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:21,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:21,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376541405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:21,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:21,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:21,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376541405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:21,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:21,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T15:41:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:21,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:21,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:21,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/b45020ca5a734331a039fbca3b1aff04 2024-11-23T15:41:21,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/2a927173e1994663bacf53d5187fab1d is 50, key is test_row_0/C:col10/1732376480599/Put/seqid=0 2024-11-23T15:41:21,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742110_1286 (size=12151) 2024-11-23T15:41:21,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-23T15:41:21,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T15:41:21,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:21,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:21,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:21,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:21,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:21,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:21,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T15:41:21,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:21,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:21,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:21,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:21,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:21,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376541904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:21,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:21,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376541904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:21,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:21,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:21,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376541914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:21,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376541914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376541914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:21,917 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:21,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/2a927173e1994663bacf53d5187fab1d 2024-11-23T15:41:21,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T15:41:21,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:21,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:21,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:21,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:21,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:21,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/9bf8e924f6394291b77d97a87ededb98 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/9bf8e924f6394291b77d97a87ededb98 2024-11-23T15:41:22,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/9bf8e924f6394291b77d97a87ededb98, entries=150, sequenceid=216, filesize=11.9 K 2024-11-23T15:41:22,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/b45020ca5a734331a039fbca3b1aff04 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/b45020ca5a734331a039fbca3b1aff04 2024-11-23T15:41:22,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/b45020ca5a734331a039fbca3b1aff04, entries=150, sequenceid=216, filesize=11.9 K 2024-11-23T15:41:22,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/2a927173e1994663bacf53d5187fab1d as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/2a927173e1994663bacf53d5187fab1d 2024-11-23T15:41:22,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/2a927173e1994663bacf53d5187fab1d, entries=150, sequenceid=216, filesize=11.9 K 2024-11-23T15:41:22,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for d2607dde3b91c49826fdae09a26d7b59 in 1294ms, sequenceid=216, compaction requested=false 2024-11-23T15:41:22,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:22,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T15:41:22,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:22,151 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-23T15:41:22,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/ec33126b2b1e403e8d9f9c438292b96f is 50, key is test_row_0/A:col10/1732376480738/Put/seqid=0 2024-11-23T15:41:22,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742111_1287 (size=12151) 2024-11-23T15:41:22,567 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/ec33126b2b1e403e8d9f9c438292b96f 2024-11-23T15:41:22,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/8591ce6f3ba7420c9810db111690d3be is 50, key is test_row_0/B:col10/1732376480738/Put/seqid=0 2024-11-23T15:41:22,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742112_1288 (size=12151) 2024-11-23T15:41:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-23T15:41:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:22,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:22,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376542941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:22,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:22,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:22,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376542942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:22,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:22,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376542942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:22,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376542943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:22,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376542945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:22,985 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/8591ce6f3ba7420c9810db111690d3be 2024-11-23T15:41:22,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/f997deec88ea47b8841afb9af47d2a2e is 50, key is test_row_0/C:col10/1732376480738/Put/seqid=0 2024-11-23T15:41:23,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742113_1289 (size=12151) 2024-11-23T15:41:23,005 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/f997deec88ea47b8841afb9af47d2a2e 2024-11-23T15:41:23,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/ec33126b2b1e403e8d9f9c438292b96f as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/ec33126b2b1e403e8d9f9c438292b96f 2024-11-23T15:41:23,020 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/ec33126b2b1e403e8d9f9c438292b96f, entries=150, sequenceid=234, filesize=11.9 K 2024-11-23T15:41:23,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/8591ce6f3ba7420c9810db111690d3be as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/8591ce6f3ba7420c9810db111690d3be 2024-11-23T15:41:23,029 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/8591ce6f3ba7420c9810db111690d3be, entries=150, sequenceid=234, filesize=11.9 K 2024-11-23T15:41:23,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/f997deec88ea47b8841afb9af47d2a2e as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/f997deec88ea47b8841afb9af47d2a2e 2024-11-23T15:41:23,051 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/f997deec88ea47b8841afb9af47d2a2e, entries=150, sequenceid=234, filesize=11.9 K 2024-11-23T15:41:23,053 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for d2607dde3b91c49826fdae09a26d7b59 in 903ms, sequenceid=234, compaction requested=true 2024-11-23T15:41:23,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:23,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:23,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-23T15:41:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-23T15:41:23,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:23,056 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:41:23,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-23T15:41:23,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-23T15:41:23,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5970 sec 2024-11-23T15:41:23,061 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.6040 sec 2024-11-23T15:41:23,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376543067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376543068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376543069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/1d9934fe18e94344904a99a3e607b6fb is 50, key is test_row_0/A:col10/1732376482934/Put/seqid=0 2024-11-23T15:41:23,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376543073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376543073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742114_1290 (size=12151) 2024-11-23T15:41:23,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/1d9934fe18e94344904a99a3e607b6fb 2024-11-23T15:41:23,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/db2c8804121144daa780d81c0646477e is 50, key is test_row_0/B:col10/1732376482934/Put/seqid=0 2024-11-23T15:41:23,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742115_1291 (size=12151) 2024-11-23T15:41:23,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/db2c8804121144daa780d81c0646477e 2024-11-23T15:41:23,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/51633d0a66f142829667f5b9847211fc is 50, key is test_row_0/C:col10/1732376482934/Put/seqid=0 2024-11-23T15:41:23,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742116_1292 (size=12151) 2024-11-23T15:41:23,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376543185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376543185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376543185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376543185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376543185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376543395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376543395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376543395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376543395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376543395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/51633d0a66f142829667f5b9847211fc 2024-11-23T15:41:23,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/1d9934fe18e94344904a99a3e607b6fb as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/1d9934fe18e94344904a99a3e607b6fb 2024-11-23T15:41:23,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/1d9934fe18e94344904a99a3e607b6fb, entries=150, sequenceid=256, filesize=11.9 K 2024-11-23T15:41:23,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/db2c8804121144daa780d81c0646477e as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/db2c8804121144daa780d81c0646477e 2024-11-23T15:41:23,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/db2c8804121144daa780d81c0646477e, entries=150, sequenceid=256, filesize=11.9 K 2024-11-23T15:41:23,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/51633d0a66f142829667f5b9847211fc as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/51633d0a66f142829667f5b9847211fc 2024-11-23T15:41:23,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/51633d0a66f142829667f5b9847211fc, entries=150, sequenceid=256, filesize=11.9 K 2024-11-23T15:41:23,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for d2607dde3b91c49826fdae09a26d7b59 in 524ms, sequenceid=256, compaction requested=true 2024-11-23T15:41:23,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:23,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:41:23,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:23,580 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:41:23,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:41:23,580 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:41:23,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:23,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:41:23,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:23,582 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:41:23,582 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/B is initiating minor compaction (all files) 2024-11-23T15:41:23,582 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/B in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:23,582 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/a260bb45be1445e09f475d9f19951e57, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/b45020ca5a734331a039fbca3b1aff04, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/8591ce6f3ba7420c9810db111690d3be, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/db2c8804121144daa780d81c0646477e] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=47.9 K 2024-11-23T15:41:23,583 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:41:23,583 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/A is initiating minor compaction (all files) 2024-11-23T15:41:23,583 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/A in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:23,583 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/58ae63165a3e4e00b4e637038051e21f, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/9bf8e924f6394291b77d97a87ededb98, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/ec33126b2b1e403e8d9f9c438292b96f, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/1d9934fe18e94344904a99a3e607b6fb] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=47.9 K 2024-11-23T15:41:23,583 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting a260bb45be1445e09f475d9f19951e57, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732376479428 2024-11-23T15:41:23,583 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58ae63165a3e4e00b4e637038051e21f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732376479428 2024-11-23T15:41:23,583 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting b45020ca5a734331a039fbca3b1aff04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732376480599 2024-11-23T15:41:23,584 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bf8e924f6394291b77d97a87ededb98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732376480599 2024-11-23T15:41:23,584 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8591ce6f3ba7420c9810db111690d3be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732376480737 2024-11-23T15:41:23,584 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec33126b2b1e403e8d9f9c438292b96f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732376480737 2024-11-23T15:41:23,584 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting db2c8804121144daa780d81c0646477e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732376482934 2024-11-23T15:41:23,585 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d9934fe18e94344904a99a3e607b6fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732376482934 2024-11-23T15:41:23,594 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#A#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:23,594 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/155f3979688c430b9a3ad5daae62d391 is 50, key is test_row_0/A:col10/1732376482934/Put/seqid=0 2024-11-23T15:41:23,597 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#B#compaction#250 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:23,598 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/6bacd0cda6d84501ac99f8d165cbaf41 is 50, key is test_row_0/B:col10/1732376482934/Put/seqid=0 2024-11-23T15:41:23,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742118_1294 (size=12731) 2024-11-23T15:41:23,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742117_1293 (size=12731) 2024-11-23T15:41:23,614 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/155f3979688c430b9a3ad5daae62d391 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/155f3979688c430b9a3ad5daae62d391 2024-11-23T15:41:23,620 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/A of d2607dde3b91c49826fdae09a26d7b59 into 155f3979688c430b9a3ad5daae62d391(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:23,620 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:23,620 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/A, priority=12, startTime=1732376483580; duration=0sec 2024-11-23T15:41:23,620 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:23,620 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:A 2024-11-23T15:41:23,621 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:41:23,622 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:41:23,622 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/C is initiating minor compaction (all files) 2024-11-23T15:41:23,622 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/C in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:23,622 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/5124158e765e410ba999d1af8ad81bbe, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/2a927173e1994663bacf53d5187fab1d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/f997deec88ea47b8841afb9af47d2a2e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/51633d0a66f142829667f5b9847211fc] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=47.9 K 2024-11-23T15:41:23,623 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5124158e765e410ba999d1af8ad81bbe, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732376479428 2024-11-23T15:41:23,623 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a927173e1994663bacf53d5187fab1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732376480599 2024-11-23T15:41:23,623 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting f997deec88ea47b8841afb9af47d2a2e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732376480737 2024-11-23T15:41:23,624 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51633d0a66f142829667f5b9847211fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732376482934 2024-11-23T15:41:23,642 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#C#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:23,644 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/678fefc1b6f04ebda48c27be509e5b67 is 50, key is test_row_0/C:col10/1732376482934/Put/seqid=0 2024-11-23T15:41:23,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742119_1295 (size=12731) 2024-11-23T15:41:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:23,710 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:41:23,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-23T15:41:23,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/6527a08773e243bba2eb145b98a0f11d is 50, key is test_row_0/A:col10/1732376483708/Put/seqid=0 2024-11-23T15:41:23,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376543742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376543746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376543748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376543749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,755 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376543750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742120_1296 (size=12301) 2024-11-23T15:41:23,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376543854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376543855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376543864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376543864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:23,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376543864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:23,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:24,007 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/6bacd0cda6d84501ac99f8d165cbaf41 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/6bacd0cda6d84501ac99f8d165cbaf41 2024-11-23T15:41:24,014 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/B of d2607dde3b91c49826fdae09a26d7b59 into 6bacd0cda6d84501ac99f8d165cbaf41(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:24,014 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:24,014 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/B, priority=12, startTime=1732376483580; duration=0sec 2024-11-23T15:41:24,014 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:24,014 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:B 2024-11-23T15:41:24,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:24,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376544065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:24,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:24,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376544065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:24,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:24,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:24,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:24,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376544074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:24,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376544074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:24,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:24,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:24,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376544074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:24,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:24,101 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/678fefc1b6f04ebda48c27be509e5b67 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/678fefc1b6f04ebda48c27be509e5b67 2024-11-23T15:41:24,106 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/C of d2607dde3b91c49826fdae09a26d7b59 into 678fefc1b6f04ebda48c27be509e5b67(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:24,106 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:24,106 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/C, priority=12, startTime=1732376483581; duration=0sec 2024-11-23T15:41:24,106 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:24,106 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:C 2024-11-23T15:41:24,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/6527a08773e243bba2eb145b98a0f11d 2024-11-23T15:41:24,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/63f927ab625a48e98873e853b0cbe0c6 is 50, key is test_row_0/B:col10/1732376483708/Put/seqid=0 2024-11-23T15:41:24,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742121_1297 (size=12301) 2024-11-23T15:41:24,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/63f927ab625a48e98873e853b0cbe0c6 2024-11-23T15:41:24,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/cdaf1d7093474be4b21ce3a2d9e307a3 is 50, key is test_row_0/C:col10/1732376483708/Put/seqid=0 2024-11-23T15:41:24,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742122_1298 (size=12301) 2024-11-23T15:41:24,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376544374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:24,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376544374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:24,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:24,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376544385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376544385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:24,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376544385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:24,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/cdaf1d7093474be4b21ce3a2d9e307a3 2024-11-23T15:41:24,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/6527a08773e243bba2eb145b98a0f11d as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6527a08773e243bba2eb145b98a0f11d 2024-11-23T15:41:24,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6527a08773e243bba2eb145b98a0f11d, entries=150, sequenceid=275, filesize=12.0 K 2024-11-23T15:41:24,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/63f927ab625a48e98873e853b0cbe0c6 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/63f927ab625a48e98873e853b0cbe0c6 2024-11-23T15:41:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-23T15:41:24,625 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:24,625 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:41:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-23T15:41:24,628 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:41:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-23T15:41:24,631 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:41:24,631 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:41:24,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/63f927ab625a48e98873e853b0cbe0c6, entries=150, sequenceid=275, filesize=12.0 K 2024-11-23T15:41:24,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/cdaf1d7093474be4b21ce3a2d9e307a3 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/cdaf1d7093474be4b21ce3a2d9e307a3 2024-11-23T15:41:24,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/cdaf1d7093474be4b21ce3a2d9e307a3, entries=150, sequenceid=275, filesize=12.0 K 2024-11-23T15:41:24,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for d2607dde3b91c49826fdae09a26d7b59 in 944ms, sequenceid=275, compaction requested=false 2024-11-23T15:41:24,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-23T15:41:24,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-23T15:41:24,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:24,785 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-23T15:41:24,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/7607ab7c35d6405cac456d5224acfd43 is 50, key is test_row_0/A:col10/1732376483749/Put/seqid=0 2024-11-23T15:41:24,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742123_1299 (size=12301) 2024-11-23T15:41:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:24,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:24,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376544904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:24,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376544904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:24,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376544907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:24,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:24,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376544908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:24,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:24,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376544908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:24,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:24,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-23T15:41:25,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:25,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376545014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:25,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:25,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:25,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376545014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:25,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376545015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376545015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:25,192 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/7607ab7c35d6405cac456d5224acfd43 2024-11-23T15:41:25,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/3011f8cc32da4eb79796c5c5c3b18df2 is 50, key is test_row_0/B:col10/1732376483749/Put/seqid=0 2024-11-23T15:41:25,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742124_1300 (size=12301) 2024-11-23T15:41:25,211 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/3011f8cc32da4eb79796c5c5c3b18df2 2024-11-23T15:41:25,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/b950c225012a4379a9a7f811a1a3e045 is 50, key is test_row_0/C:col10/1732376483749/Put/seqid=0 2024-11-23T15:41:25,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:25,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376545225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:25,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:25,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:25,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376545225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:25,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:25,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376545225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:25,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:25,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376545225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:25,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:25,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,229 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:25,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:25,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742125_1301 (size=12301) 2024-11-23T15:41:25,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-23T15:41:25,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376545535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:25,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:25,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376545535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376545536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:25,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376545537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:25,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:25,641 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/b950c225012a4379a9a7f811a1a3e045 2024-11-23T15:41:25,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/7607ab7c35d6405cac456d5224acfd43 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/7607ab7c35d6405cac456d5224acfd43 2024-11-23T15:41:25,658 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/7607ab7c35d6405cac456d5224acfd43, entries=150, sequenceid=295, filesize=12.0 K 2024-11-23T15:41:25,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/3011f8cc32da4eb79796c5c5c3b18df2 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/3011f8cc32da4eb79796c5c5c3b18df2 2024-11-23T15:41:25,666 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/3011f8cc32da4eb79796c5c5c3b18df2, entries=150, sequenceid=295, filesize=12.0 K 2024-11-23T15:41:25,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/b950c225012a4379a9a7f811a1a3e045 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/b950c225012a4379a9a7f811a1a3e045 2024-11-23T15:41:25,671 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/b950c225012a4379a9a7f811a1a3e045, entries=150, sequenceid=295, filesize=12.0 K 2024-11-23T15:41:25,672 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for d2607dde3b91c49826fdae09a26d7b59 in 888ms, sequenceid=295, compaction requested=true 2024-11-23T15:41:25,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:25,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:25,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-23T15:41:25,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-23T15:41:25,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-23T15:41:25,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0420 sec 2024-11-23T15:41:25,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.0520 sec 2024-11-23T15:41:25,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-23T15:41:25,771 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:25,771 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:41:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-23T15:41:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-23T15:41:25,774 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:41:25,775 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:41:25,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:41:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-23T15:41:25,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:25,916 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:41:25,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-23T15:41:25,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/770e7d31d8834b79adc940d400ed28f2 is 50, key is test_row_0/A:col10/1732376485915/Put/seqid=0 2024-11-23T15:41:25,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-23T15:41:25,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:25,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:25,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:25,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:25,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:25,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:25,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742126_1302 (size=14741) 2024-11-23T15:41:25,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376545980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:25,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:26,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376546044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:26,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376546044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:26,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376546045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:26,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376546045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:26,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-23T15:41:26,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:26,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:26,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:26,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:26,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-23T15:41:26,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376546094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:26,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-23T15:41:26,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:26,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:26,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:26,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:26,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:26,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376546304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:26,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:26,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/770e7d31d8834b79adc940d400ed28f2 2024-11-23T15:41:26,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/3acd34999566408098c9b4bf9ca9705e is 50, key is test_row_0/B:col10/1732376485915/Put/seqid=0 2024-11-23T15:41:26,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742127_1303 (size=12301) 2024-11-23T15:41:26,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/3acd34999566408098c9b4bf9ca9705e 2024-11-23T15:41:26,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/b3cc66987b5c404e8f5a7bf488c60054 is 50, key is test_row_0/C:col10/1732376485915/Put/seqid=0 2024-11-23T15:41:26,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742128_1304 (size=12301) 2024-11-23T15:41:26,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-23T15:41:26,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:26,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:26,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:26,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:26,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-23T15:41:26,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-23T15:41:26,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:26,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:26,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:26,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:26,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:26,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376546614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:26,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:26,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:26,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:26,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-23T15:41:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:26,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:26,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/b3cc66987b5c404e8f5a7bf488c60054 2024-11-23T15:41:26,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/770e7d31d8834b79adc940d400ed28f2 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/770e7d31d8834b79adc940d400ed28f2 2024-11-23T15:41:26,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/770e7d31d8834b79adc940d400ed28f2, entries=200, sequenceid=315, filesize=14.4 K 2024-11-23T15:41:26,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/3acd34999566408098c9b4bf9ca9705e as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/3acd34999566408098c9b4bf9ca9705e 2024-11-23T15:41:26,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/3acd34999566408098c9b4bf9ca9705e, entries=150, sequenceid=315, filesize=12.0 K 2024-11-23T15:41:26,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/b3cc66987b5c404e8f5a7bf488c60054 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/b3cc66987b5c404e8f5a7bf488c60054 2024-11-23T15:41:26,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/b3cc66987b5c404e8f5a7bf488c60054, entries=150, sequenceid=315, filesize=12.0 K 2024-11-23T15:41:26,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for d2607dde3b91c49826fdae09a26d7b59 in 860ms, sequenceid=315, compaction requested=true 2024-11-23T15:41:26,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:26,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:41:26,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:26,777 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:41:26,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:41:26,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:26,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:41:26,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-23T15:41:26,777 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:41:26,778 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:41:26,778 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52074 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:41:26,778 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/B is initiating minor compaction (all files) 2024-11-23T15:41:26,778 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/A is initiating minor compaction (all files) 2024-11-23T15:41:26,778 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/B in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:26,778 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/A in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:26,778 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/6bacd0cda6d84501ac99f8d165cbaf41, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/63f927ab625a48e98873e853b0cbe0c6, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/3011f8cc32da4eb79796c5c5c3b18df2, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/3acd34999566408098c9b4bf9ca9705e] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=48.5 K 2024-11-23T15:41:26,778 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/155f3979688c430b9a3ad5daae62d391, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6527a08773e243bba2eb145b98a0f11d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/7607ab7c35d6405cac456d5224acfd43, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/770e7d31d8834b79adc940d400ed28f2] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=50.9 K 2024-11-23T15:41:26,778 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 155f3979688c430b9a3ad5daae62d391, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732376482934 2024-11-23T15:41:26,779 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bacd0cda6d84501ac99f8d165cbaf41, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732376482934 2024-11-23T15:41:26,779 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6527a08773e243bba2eb145b98a0f11d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732376483072 2024-11-23T15:41:26,779 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 63f927ab625a48e98873e853b0cbe0c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732376483072 2024-11-23T15:41:26,779 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7607ab7c35d6405cac456d5224acfd43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732376483739 2024-11-23T15:41:26,779 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 3011f8cc32da4eb79796c5c5c3b18df2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732376483739 2024-11-23T15:41:26,779 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 770e7d31d8834b79adc940d400ed28f2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732376484900 2024-11-23T15:41:26,779 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 3acd34999566408098c9b4bf9ca9705e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732376484900 2024-11-23T15:41:26,791 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#B#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:26,791 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/fc0d965933594b05b2764a89d208328e is 50, key is test_row_0/B:col10/1732376485915/Put/seqid=0 2024-11-23T15:41:26,792 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#A#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:26,793 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/b36e35cbbbc54844b4d2d3b470e3fd59 is 50, key is test_row_0/A:col10/1732376485915/Put/seqid=0 2024-11-23T15:41:26,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742129_1305 (size=13017) 2024-11-23T15:41:26,813 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/b36e35cbbbc54844b4d2d3b470e3fd59 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/b36e35cbbbc54844b4d2d3b470e3fd59 2024-11-23T15:41:26,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742130_1306 (size=13017) 2024-11-23T15:41:26,821 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/fc0d965933594b05b2764a89d208328e as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/fc0d965933594b05b2764a89d208328e 2024-11-23T15:41:26,822 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/A of d2607dde3b91c49826fdae09a26d7b59 into b36e35cbbbc54844b4d2d3b470e3fd59(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:26,822 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:26,822 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/A, priority=12, startTime=1732376486777; duration=0sec 2024-11-23T15:41:26,822 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:26,822 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:A 2024-11-23T15:41:26,822 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:41:26,825 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:41:26,825 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/C is initiating minor compaction (all files) 2024-11-23T15:41:26,825 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/C in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:26,825 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/678fefc1b6f04ebda48c27be509e5b67, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/cdaf1d7093474be4b21ce3a2d9e307a3, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/b950c225012a4379a9a7f811a1a3e045, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/b3cc66987b5c404e8f5a7bf488c60054] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=48.5 K 2024-11-23T15:41:26,826 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 678fefc1b6f04ebda48c27be509e5b67, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732376482934 2024-11-23T15:41:26,827 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdaf1d7093474be4b21ce3a2d9e307a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732376483072 2024-11-23T15:41:26,828 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting b950c225012a4379a9a7f811a1a3e045, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732376483739 2024-11-23T15:41:26,828 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/B of d2607dde3b91c49826fdae09a26d7b59 into fc0d965933594b05b2764a89d208328e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:26,828 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:26,828 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/B, priority=12, startTime=1732376486777; duration=0sec 2024-11-23T15:41:26,828 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:26,828 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:B 2024-11-23T15:41:26,828 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3cc66987b5c404e8f5a7bf488c60054, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732376484900 2024-11-23T15:41:26,837 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#C#compaction#263 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:26,837 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/54586241b63d43f9afcb8e62a6575d55 is 50, key is test_row_0/C:col10/1732376485915/Put/seqid=0 2024-11-23T15:41:26,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-23T15:41:26,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:26,847 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-23T15:41:26,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/fb603ef9af57484cb2157c7600ddb9d7 is 50, key is test_row_0/A:col10/1732376485978/Put/seqid=0 2024-11-23T15:41:26,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742131_1307 (size=13017) 2024-11-23T15:41:26,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742132_1308 (size=12301) 2024-11-23T15:41:26,869 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/fb603ef9af57484cb2157c7600ddb9d7 2024-11-23T15:41:26,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/bfe94dfb623e4170af285849a85bc532 is 50, key is test_row_0/B:col10/1732376485978/Put/seqid=0 2024-11-23T15:41:26,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742133_1309 (size=12301) 2024-11-23T15:41:26,883 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/bfe94dfb623e4170af285849a85bc532 2024-11-23T15:41:26,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/5c904bc1363a4294bb09fa0e18a628cf is 50, key is test_row_0/C:col10/1732376485978/Put/seqid=0 2024-11-23T15:41:26,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-23T15:41:26,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742134_1310 (size=12301) 2024-11-23T15:41:26,938 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/5c904bc1363a4294bb09fa0e18a628cf 2024-11-23T15:41:26,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/fb603ef9af57484cb2157c7600ddb9d7 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/fb603ef9af57484cb2157c7600ddb9d7 2024-11-23T15:41:26,954 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/fb603ef9af57484cb2157c7600ddb9d7, entries=150, sequenceid=333, filesize=12.0 K 2024-11-23T15:41:26,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/bfe94dfb623e4170af285849a85bc532 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bfe94dfb623e4170af285849a85bc532 2024-11-23T15:41:26,959 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bfe94dfb623e4170af285849a85bc532, entries=150, sequenceid=333, filesize=12.0 K 2024-11-23T15:41:26,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/5c904bc1363a4294bb09fa0e18a628cf as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/5c904bc1363a4294bb09fa0e18a628cf 2024-11-23T15:41:26,963 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/5c904bc1363a4294bb09fa0e18a628cf, entries=150, sequenceid=333, filesize=12.0 K 2024-11-23T15:41:26,965 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=0 B/0 for d2607dde3b91c49826fdae09a26d7b59 in 118ms, sequenceid=333, compaction requested=false 2024-11-23T15:41:26,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:26,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:26,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-23T15:41:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-23T15:41:26,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-23T15:41:26,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1910 sec 2024-11-23T15:41:26,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.1990 sec 2024-11-23T15:41:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:27,063 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:41:27,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T15:41:27,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/d1739b2dcba14483b9e22b5d777fcbec is 50, key is test_row_0/A:col10/1732376487062/Put/seqid=0 2024-11-23T15:41:27,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742135_1311 (size=12301) 2024-11-23T15:41:27,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:27,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376547095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:27,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376547096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:27,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376547098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:27,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376547101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:27,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:27,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:27,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376547124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:27,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:27,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376547204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:27,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:27,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376547204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:27,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:27,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376547214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:27,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:27,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376547214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:27,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:27,265 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/54586241b63d43f9afcb8e62a6575d55 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/54586241b63d43f9afcb8e62a6575d55 2024-11-23T15:41:27,270 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/C of d2607dde3b91c49826fdae09a26d7b59 into 54586241b63d43f9afcb8e62a6575d55(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:27,270 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:27,270 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/C, priority=12, startTime=1732376486777; duration=0sec 2024-11-23T15:41:27,270 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:27,270 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:C 2024-11-23T15:41:27,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:27,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376547414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376547414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:27,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376547424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:27,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376547424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:27,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:27,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:27,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/d1739b2dcba14483b9e22b5d777fcbec 2024-11-23T15:41:27,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/bb1da1c161fd4d49a63fbd7fcd7a6ced is 50, key is test_row_0/B:col10/1732376487062/Put/seqid=0 2024-11-23T15:41:27,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742136_1312 (size=12301) 2024-11-23T15:41:27,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:27,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376547724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:27,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:27,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376547724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:27,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:27,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:27,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376547734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:27,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:27,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376547739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:27,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:27,742 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:27,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/bb1da1c161fd4d49a63fbd7fcd7a6ced 2024-11-23T15:41:27,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/dbd492977d62471b9ba004acb92c9e83 is 50, key is test_row_0/C:col10/1732376487062/Put/seqid=0 2024-11-23T15:41:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-23T15:41:27,924 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:27,924 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:41:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-23T15:41:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-23T15:41:27,926 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:41:27,927 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:41:27,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:41:27,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742137_1313 (size=12301) 2024-11-23T15:41:28,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-23T15:41:28,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-23T15:41:28,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:28,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:28,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:28,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:28,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:28,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376548134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:28,138 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_0' in d2607dde3b91c49826fdae09a26d7b59 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:41:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:28,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-23T15:41:28,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:28,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:28,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:28,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:28,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:28,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:28,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376548234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:28,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:28,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:28,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376548235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:28,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:28,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:28,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:28,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:28,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-23T15:41:28,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376548244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:28,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:28,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:28,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:28,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:28,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376548244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:28,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:28,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:28,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:28,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/dbd492977d62471b9ba004acb92c9e83 2024-11-23T15:41:28,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/d1739b2dcba14483b9e22b5d777fcbec as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/d1739b2dcba14483b9e22b5d777fcbec 2024-11-23T15:41:28,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/d1739b2dcba14483b9e22b5d777fcbec, entries=150, sequenceid=344, filesize=12.0 K 2024-11-23T15:41:28,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/bb1da1c161fd4d49a63fbd7fcd7a6ced as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bb1da1c161fd4d49a63fbd7fcd7a6ced 2024-11-23T15:41:28,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bb1da1c161fd4d49a63fbd7fcd7a6ced, entries=150, sequenceid=344, filesize=12.0 K 2024-11-23T15:41:28,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/dbd492977d62471b9ba004acb92c9e83 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/dbd492977d62471b9ba004acb92c9e83 2024-11-23T15:41:28,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/dbd492977d62471b9ba004acb92c9e83, entries=150, sequenceid=344, filesize=12.0 K 2024-11-23T15:41:28,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d2607dde3b91c49826fdae09a26d7b59 in 1289ms, sequenceid=344, compaction requested=true 2024-11-23T15:41:28,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:28,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:41:28,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:28,352 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:28,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:41:28,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:28,352 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:28,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:41:28,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:28,353 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:28,353 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:28,353 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/B is initiating minor compaction (all files) 2024-11-23T15:41:28,353 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/A is initiating minor compaction (all files) 2024-11-23T15:41:28,353 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/A in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:28,353 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/B in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:28,353 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/b36e35cbbbc54844b4d2d3b470e3fd59, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/fb603ef9af57484cb2157c7600ddb9d7, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/d1739b2dcba14483b9e22b5d777fcbec] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=36.7 K 2024-11-23T15:41:28,353 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/fc0d965933594b05b2764a89d208328e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bfe94dfb623e4170af285849a85bc532, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bb1da1c161fd4d49a63fbd7fcd7a6ced] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=36.7 K 2024-11-23T15:41:28,353 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting b36e35cbbbc54844b4d2d3b470e3fd59, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732376484900 2024-11-23T15:41:28,353 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting fc0d965933594b05b2764a89d208328e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732376484900 2024-11-23T15:41:28,354 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting bfe94dfb623e4170af285849a85bc532, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732376485967 2024-11-23T15:41:28,354 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb603ef9af57484cb2157c7600ddb9d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732376485967 2024-11-23T15:41:28,354 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting bb1da1c161fd4d49a63fbd7fcd7a6ced, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732376487060 2024-11-23T15:41:28,354 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1739b2dcba14483b9e22b5d777fcbec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732376487060 2024-11-23T15:41:28,361 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#A#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:28,362 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/3820563835304856a3ec866bdaeadcfc is 50, key is test_row_0/A:col10/1732376487062/Put/seqid=0 2024-11-23T15:41:28,362 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#B#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:28,363 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/bf5a6ffe6e1848c2a0e1fd76015f8d9b is 50, key is test_row_0/B:col10/1732376487062/Put/seqid=0 2024-11-23T15:41:28,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742138_1314 (size=13119) 2024-11-23T15:41:28,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742139_1315 (size=13119) 2024-11-23T15:41:28,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-23T15:41:28,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:28,385 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T15:41:28,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/0bd9f8d861814843881a081b256da168 is 50, key is test_row_0/A:col10/1732376487096/Put/seqid=0 2024-11-23T15:41:28,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742140_1316 (size=12301) 2024-11-23T15:41:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-23T15:41:28,771 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/3820563835304856a3ec866bdaeadcfc as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/3820563835304856a3ec866bdaeadcfc 2024-11-23T15:41:28,775 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/bf5a6ffe6e1848c2a0e1fd76015f8d9b as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bf5a6ffe6e1848c2a0e1fd76015f8d9b 2024-11-23T15:41:28,786 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/B of d2607dde3b91c49826fdae09a26d7b59 into bf5a6ffe6e1848c2a0e1fd76015f8d9b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:28,787 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:28,787 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/B, priority=13, startTime=1732376488352; duration=0sec 2024-11-23T15:41:28,787 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:28,787 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:B 2024-11-23T15:41:28,787 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:28,787 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/A of d2607dde3b91c49826fdae09a26d7b59 into 3820563835304856a3ec866bdaeadcfc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:28,787 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:28,787 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/A, priority=13, startTime=1732376488352; duration=0sec 2024-11-23T15:41:28,787 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:28,787 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:A 2024-11-23T15:41:28,789 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:28,789 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/C is initiating minor compaction (all files) 2024-11-23T15:41:28,789 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/C in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:28,789 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/54586241b63d43f9afcb8e62a6575d55, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/5c904bc1363a4294bb09fa0e18a628cf, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/dbd492977d62471b9ba004acb92c9e83] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=36.7 K 2024-11-23T15:41:28,801 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 54586241b63d43f9afcb8e62a6575d55, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732376484900 2024-11-23T15:41:28,801 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/0bd9f8d861814843881a081b256da168 2024-11-23T15:41:28,804 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c904bc1363a4294bb09fa0e18a628cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1732376485967 2024-11-23T15:41:28,805 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting dbd492977d62471b9ba004acb92c9e83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732376487060 2024-11-23T15:41:28,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/ee18f23db8eb4eb6aafe2e4351177541 is 50, key is test_row_0/B:col10/1732376487096/Put/seqid=0 2024-11-23T15:41:28,833 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#C#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:28,833 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/22d60596680d4ed999485d9bb806c7fd is 50, key is test_row_0/C:col10/1732376487062/Put/seqid=0 2024-11-23T15:41:28,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742141_1317 (size=12301) 2024-11-23T15:41:28,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742142_1318 (size=13119) 2024-11-23T15:41:28,878 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/22d60596680d4ed999485d9bb806c7fd as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/22d60596680d4ed999485d9bb806c7fd 2024-11-23T15:41:28,904 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/C of d2607dde3b91c49826fdae09a26d7b59 into 22d60596680d4ed999485d9bb806c7fd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:28,904 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:28,904 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/C, priority=13, startTime=1732376488352; duration=0sec 2024-11-23T15:41:28,904 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:28,904 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:C 2024-11-23T15:41:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-23T15:41:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:29,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:29,254 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/ee18f23db8eb4eb6aafe2e4351177541 2024-11-23T15:41:29,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376549256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:29,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:29,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376549257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:29,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376549260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:29,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:29,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/b5ec23e4db3f4740ada1c54aa95d13c9 is 50, key is test_row_0/C:col10/1732376487096/Put/seqid=0 2024-11-23T15:41:29,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376549261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:29,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742143_1319 (size=12301) 2024-11-23T15:41:29,267 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/b5ec23e4db3f4740ada1c54aa95d13c9 2024-11-23T15:41:29,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/0bd9f8d861814843881a081b256da168 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/0bd9f8d861814843881a081b256da168 2024-11-23T15:41:29,278 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/0bd9f8d861814843881a081b256da168, entries=150, sequenceid=370, filesize=12.0 K 2024-11-23T15:41:29,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/ee18f23db8eb4eb6aafe2e4351177541 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/ee18f23db8eb4eb6aafe2e4351177541 2024-11-23T15:41:29,287 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/ee18f23db8eb4eb6aafe2e4351177541, entries=150, sequenceid=370, filesize=12.0 K 2024-11-23T15:41:29,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/b5ec23e4db3f4740ada1c54aa95d13c9 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/b5ec23e4db3f4740ada1c54aa95d13c9 2024-11-23T15:41:29,291 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/b5ec23e4db3f4740ada1c54aa95d13c9, entries=150, sequenceid=370, filesize=12.0 K 2024-11-23T15:41:29,292 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d2607dde3b91c49826fdae09a26d7b59 in 907ms, sequenceid=370, compaction requested=false 2024-11-23T15:41:29,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:29,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:29,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-23T15:41:29,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-23T15:41:29,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-23T15:41:29,295 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3660 sec 2024-11-23T15:41:29,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.3720 sec 2024-11-23T15:41:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:29,366 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:41:29,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T15:41:29,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/1cb851247d1d414994eec6b001d9fbf0 is 50, key is test_row_0/A:col10/1732376489365/Put/seqid=0 2024-11-23T15:41:29,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742144_1320 (size=9857) 2024-11-23T15:41:29,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376549390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:29,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376549391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:29,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:29,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:29,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376549394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:29,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:29,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:29,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376549395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:29,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:29,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376549504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:29,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:29,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376549504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:29,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376549504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:29,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:29,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376549504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:29,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376549714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:29,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376549714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376549714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376549714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:29,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:29,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/1cb851247d1d414994eec6b001d9fbf0 2024-11-23T15:41:29,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/f0ccf2f9731a44e0a04b67f1a526d200 is 50, key is test_row_0/B:col10/1732376489365/Put/seqid=0 2024-11-23T15:41:29,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742145_1321 (size=9857) 2024-11-23T15:41:30,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376550024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:30,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376550024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:30,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:30,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376550024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:30,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:30,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376550024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:30,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:30,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-23T15:41:30,075 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:30,075 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:41:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-23T15:41:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-23T15:41:30,078 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:41:30,080 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:41:30,080 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:41:30,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:30,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34106 deadline: 1732376550154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:30,157 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_0' in d2607dde3b91c49826fdae09a26d7b59 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:41:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:30,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:30,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-23T15:41:30,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/f0ccf2f9731a44e0a04b67f1a526d200 2024-11-23T15:41:30,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/729249a42fa143a59582d5cbf8a6396c is 50, key is test_row_0/C:col10/1732376489365/Put/seqid=0 2024-11-23T15:41:30,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742146_1322 (size=9857) 2024-11-23T15:41:30,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-23T15:41:30,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:30,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:30,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:30,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:30,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:30,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:30,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-23T15:41:30,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:30,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:30,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:30,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:30,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:30,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:30,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-23T15:41:30,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:30,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376550534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:30,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376550534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:30,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376550534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:30,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376550534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:30,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-23T15:41:30,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:30,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:30,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:30,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:30,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:30,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/729249a42fa143a59582d5cbf8a6396c 2024-11-23T15:41:30,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/1cb851247d1d414994eec6b001d9fbf0 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/1cb851247d1d414994eec6b001d9fbf0 2024-11-23T15:41:30,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/1cb851247d1d414994eec6b001d9fbf0, entries=100, sequenceid=387, filesize=9.6 K 2024-11-23T15:41:30,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/f0ccf2f9731a44e0a04b67f1a526d200 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/f0ccf2f9731a44e0a04b67f1a526d200 2024-11-23T15:41:30,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/f0ccf2f9731a44e0a04b67f1a526d200, entries=100, sequenceid=387, filesize=9.6 K 2024-11-23T15:41:30,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/729249a42fa143a59582d5cbf8a6396c as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/729249a42fa143a59582d5cbf8a6396c 2024-11-23T15:41:30,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/729249a42fa143a59582d5cbf8a6396c, entries=100, sequenceid=387, filesize=9.6 K 2024-11-23T15:41:30,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d2607dde3b91c49826fdae09a26d7b59 in 1252ms, sequenceid=387, compaction requested=true 2024-11-23T15:41:30,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:30,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:41:30,618 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:30,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:30,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:41:30,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:30,619 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:30,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:41:30,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:30,619 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:30,619 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:30,619 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/A is initiating minor compaction (all files) 2024-11-23T15:41:30,619 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/B is initiating minor compaction (all files) 2024-11-23T15:41:30,619 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/B in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:30,619 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/A in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:30,620 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bf5a6ffe6e1848c2a0e1fd76015f8d9b, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/ee18f23db8eb4eb6aafe2e4351177541, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/f0ccf2f9731a44e0a04b67f1a526d200] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=34.5 K 2024-11-23T15:41:30,620 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/3820563835304856a3ec866bdaeadcfc, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/0bd9f8d861814843881a081b256da168, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/1cb851247d1d414994eec6b001d9fbf0] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=34.5 K 2024-11-23T15:41:30,620 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting bf5a6ffe6e1848c2a0e1fd76015f8d9b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732376487060 2024-11-23T15:41:30,620 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3820563835304856a3ec866bdaeadcfc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732376487060 2024-11-23T15:41:30,620 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting ee18f23db8eb4eb6aafe2e4351177541, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732376487091 2024-11-23T15:41:30,620 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bd9f8d861814843881a081b256da168, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732376487091 2024-11-23T15:41:30,620 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cb851247d1d414994eec6b001d9fbf0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732376489258 2024-11-23T15:41:30,620 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting f0ccf2f9731a44e0a04b67f1a526d200, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732376489258 2024-11-23T15:41:30,627 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#B#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:30,628 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/1388c296dc0c41609dd4a9b8175b44fb is 50, key is test_row_0/B:col10/1732376489365/Put/seqid=0 2024-11-23T15:41:30,628 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#A#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:30,629 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/28c52e7a8c294ef4b5d640437752f930 is 50, key is test_row_0/A:col10/1732376489365/Put/seqid=0 2024-11-23T15:41:30,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742148_1324 (size=13221) 2024-11-23T15:41:30,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742147_1323 (size=13221) 2024-11-23T15:41:30,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-23T15:41:30,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:30,691 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-23T15:41:30,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/34d159b362ae4c32b616855049913011 is 50, key is test_row_0/A:col10/1732376489389/Put/seqid=0 2024-11-23T15:41:30,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742149_1325 (size=12301) 2024-11-23T15:41:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-23T15:41:31,035 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/28c52e7a8c294ef4b5d640437752f930 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/28c52e7a8c294ef4b5d640437752f930 2024-11-23T15:41:31,036 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/1388c296dc0c41609dd4a9b8175b44fb as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/1388c296dc0c41609dd4a9b8175b44fb 2024-11-23T15:41:31,040 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/B of d2607dde3b91c49826fdae09a26d7b59 into 1388c296dc0c41609dd4a9b8175b44fb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:31,040 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:31,040 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/B, priority=13, startTime=1732376490618; duration=0sec 2024-11-23T15:41:31,040 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:31,040 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:B 2024-11-23T15:41:31,040 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/A of d2607dde3b91c49826fdae09a26d7b59 into 28c52e7a8c294ef4b5d640437752f930(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:31,040 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:31,040 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/A, priority=13, startTime=1732376490618; duration=0sec 2024-11-23T15:41:31,040 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:31,041 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:31,041 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:A 2024-11-23T15:41:31,041 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:31,042 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/C is initiating minor compaction (all files) 2024-11-23T15:41:31,042 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/C in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:31,042 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/22d60596680d4ed999485d9bb806c7fd, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/b5ec23e4db3f4740ada1c54aa95d13c9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/729249a42fa143a59582d5cbf8a6396c] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=34.5 K 2024-11-23T15:41:31,042 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 22d60596680d4ed999485d9bb806c7fd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732376487060 2024-11-23T15:41:31,042 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting b5ec23e4db3f4740ada1c54aa95d13c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732376487091 2024-11-23T15:41:31,042 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 729249a42fa143a59582d5cbf8a6396c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732376489258 2024-11-23T15:41:31,048 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#C#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:31,049 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/50e6628e28f04a839fd7f83e547f3b8c is 50, key is test_row_0/C:col10/1732376489365/Put/seqid=0 2024-11-23T15:41:31,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742150_1326 (size=13221) 2024-11-23T15:41:31,100 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/34d159b362ae4c32b616855049913011 2024-11-23T15:41:31,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/babb3189270b48b98b71a23e33f3e925 is 50, key is test_row_0/B:col10/1732376489389/Put/seqid=0 2024-11-23T15:41:31,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742151_1327 (size=12301) 2024-11-23T15:41:31,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-23T15:41:31,456 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/50e6628e28f04a839fd7f83e547f3b8c as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/50e6628e28f04a839fd7f83e547f3b8c 2024-11-23T15:41:31,461 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/C of d2607dde3b91c49826fdae09a26d7b59 into 50e6628e28f04a839fd7f83e547f3b8c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:31,461 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:31,461 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/C, priority=13, startTime=1732376490619; duration=0sec 2024-11-23T15:41:31,461 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:31,461 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:C 2024-11-23T15:41:31,516 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/babb3189270b48b98b71a23e33f3e925 2024-11-23T15:41:31,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/59597fda80eb4417a1825419fb3d5de0 is 50, key is test_row_0/C:col10/1732376489389/Put/seqid=0 2024-11-23T15:41:31,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742152_1328 (size=12301) 2024-11-23T15:41:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:31,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:31,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:31,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376551584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376551585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:31,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376551586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:31,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:31,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:31,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376551586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:31,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:31,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:31,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376551694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:31,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:31,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:31,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376551694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376551694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:31,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376551694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:31,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:31,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376551904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:31,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:31,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376551904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:31,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376551904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:31,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:31,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376551904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:31,936 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/59597fda80eb4417a1825419fb3d5de0 2024-11-23T15:41:31,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/34d159b362ae4c32b616855049913011 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/34d159b362ae4c32b616855049913011 2024-11-23T15:41:31,943 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/34d159b362ae4c32b616855049913011, entries=150, sequenceid=409, filesize=12.0 K 2024-11-23T15:41:31,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/babb3189270b48b98b71a23e33f3e925 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/babb3189270b48b98b71a23e33f3e925 2024-11-23T15:41:31,947 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/babb3189270b48b98b71a23e33f3e925, entries=150, sequenceid=409, filesize=12.0 K 2024-11-23T15:41:31,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/59597fda80eb4417a1825419fb3d5de0 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/59597fda80eb4417a1825419fb3d5de0 2024-11-23T15:41:31,952 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/59597fda80eb4417a1825419fb3d5de0, entries=150, sequenceid=409, filesize=12.0 K 2024-11-23T15:41:31,953 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for d2607dde3b91c49826fdae09a26d7b59 in 1262ms, sequenceid=409, compaction requested=false 2024-11-23T15:41:31,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:31,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:31,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-23T15:41:31,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-23T15:41:31,956 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-23T15:41:31,956 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8750 sec 2024-11-23T15:41:31,958 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 1.8820 sec 2024-11-23T15:41:32,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:32,217 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:41:32,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-23T15:41:32,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/6a174b6c6c734bbdbd41c6b0ab72ae33 is 50, key is test_row_0/A:col10/1732376491585/Put/seqid=0 2024-11-23T15:41:32,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742153_1329 (size=17181) 2024-11-23T15:41:32,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-23T15:41:32,234 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:32,235 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:41:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-23T15:41:32,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-23T15:41:32,236 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:41:32,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:32,237 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:41:32,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376552233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:32,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:41:32,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:32,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:32,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376552234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:32,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376552237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:32,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376552237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,241 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-23T15:41:32,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:32,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376552344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:32,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:32,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:32,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376552344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:32,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376552344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:32,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:32,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376552345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:32,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-23T15:41:32,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:32,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:32,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:32,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:32,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:32,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:32,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-23T15:41:32,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:32,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:32,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:32,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:32,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:32,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:32,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-23T15:41:32,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:32,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376552554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:32,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376552554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:32,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376552554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:32,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376552554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:32,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/6a174b6c6c734bbdbd41c6b0ab72ae33 2024-11-23T15:41:32,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/bad58d697fae4e6f943bb8be623289dc is 50, key is test_row_0/B:col10/1732376491585/Put/seqid=0 2024-11-23T15:41:32,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742154_1330 (size=12301) 2024-11-23T15:41:32,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-23T15:41:32,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:32,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:32,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:32,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:32,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:32,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:32,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-23T15:41:32,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:32,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:32,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:32,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:32,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:32,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:32,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-23T15:41:32,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:32,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376552864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:32,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:32,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376552864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:32,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:32,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376552864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:32,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:32,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:32,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376552864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:32,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:32,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:32,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-23T15:41:32,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:32,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:32,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:32,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:33,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:33,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/bad58d697fae4e6f943bb8be623289dc 2024-11-23T15:41:33,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/aba21285a82241a884c374e3c7d3aea9 is 50, key is test_row_0/C:col10/1732376491585/Put/seqid=0 2024-11-23T15:41:33,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742155_1331 (size=12301) 2024-11-23T15:41:33,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-23T15:41:33,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:33,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:33,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:33,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:33,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:33,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:33,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-23T15:41:33,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:33,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:33,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:33,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:33,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:33,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:33,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-23T15:41:33,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:33,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34148 deadline: 1732376553374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:33,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:33,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34164 deadline: 1732376553375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:33,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:33,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:33,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:33,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:33,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:33,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:33,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:33,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34122 deadline: 1732376553375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:33,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:33,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:33,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34184 deadline: 1732376553375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:33,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:33,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:33,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:33,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:33,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:41:33,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/aba21285a82241a884c374e3c7d3aea9 2024-11-23T15:41:33,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/6a174b6c6c734bbdbd41c6b0ab72ae33 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6a174b6c6c734bbdbd41c6b0ab72ae33 2024-11-23T15:41:33,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6a174b6c6c734bbdbd41c6b0ab72ae33, entries=250, sequenceid=430, filesize=16.8 K 2024-11-23T15:41:33,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/bad58d697fae4e6f943bb8be623289dc as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bad58d697fae4e6f943bb8be623289dc 2024-11-23T15:41:33,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bad58d697fae4e6f943bb8be623289dc, entries=150, sequenceid=430, filesize=12.0 K 2024-11-23T15:41:33,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-23T15:41:33,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:33,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:33,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:33,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/aba21285a82241a884c374e3c7d3aea9 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/aba21285a82241a884c374e3c7d3aea9 2024-11-23T15:41:33,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:33,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:33,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:33,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/aba21285a82241a884c374e3c7d3aea9, entries=150, sequenceid=430, filesize=12.0 K 2024-11-23T15:41:33,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for d2607dde3b91c49826fdae09a26d7b59 in 1248ms, sequenceid=430, compaction requested=true 2024-11-23T15:41:33,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:33,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:41:33,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:33,465 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:33,466 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:33,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:41:33,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:33,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d2607dde3b91c49826fdae09a26d7b59:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:41:33,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:33,468 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:33,468 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:33,468 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/B is initiating minor compaction (all files) 2024-11-23T15:41:33,468 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/A is initiating minor compaction (all files) 2024-11-23T15:41:33,469 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/B in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:33,469 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/A in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:33,469 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/1388c296dc0c41609dd4a9b8175b44fb, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/babb3189270b48b98b71a23e33f3e925, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bad58d697fae4e6f943bb8be623289dc] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=36.9 K 2024-11-23T15:41:33,469 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/28c52e7a8c294ef4b5d640437752f930, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/34d159b362ae4c32b616855049913011, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6a174b6c6c734bbdbd41c6b0ab72ae33] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=41.7 K 2024-11-23T15:41:33,469 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 1388c296dc0c41609dd4a9b8175b44fb, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732376487099 2024-11-23T15:41:33,469 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28c52e7a8c294ef4b5d640437752f930, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732376487099 2024-11-23T15:41:33,469 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34d159b362ae4c32b616855049913011, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732376489387 2024-11-23T15:41:33,469 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting babb3189270b48b98b71a23e33f3e925, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732376489387 2024-11-23T15:41:33,469 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a174b6c6c734bbdbd41c6b0ab72ae33, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732376491557 2024-11-23T15:41:33,469 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting bad58d697fae4e6f943bb8be623289dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732376491557 2024-11-23T15:41:33,475 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#A#compaction#289 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:33,476 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/23b5fbe576eb45ff9a90ab6636a68c8d is 50, key is test_row_0/A:col10/1732376491585/Put/seqid=0 2024-11-23T15:41:33,476 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#B#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:33,477 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/302fe0b0d9d94177aed9e95e4da03d03 is 50, key is test_row_0/B:col10/1732376491585/Put/seqid=0 2024-11-23T15:41:33,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742157_1333 (size=13323) 2024-11-23T15:41:33,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742156_1332 (size=13323) 2024-11-23T15:41:33,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-23T15:41:33,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:33,611 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-23T15:41:33,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/a47e9182d0214a538c386beb5efe6dc8 is 50, key is test_row_0/A:col10/1732376492236/Put/seqid=0 2024-11-23T15:41:33,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742158_1334 (size=12301) 2024-11-23T15:41:33,874 INFO [Thread-1140 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1140. 2024-11-23T15:41:33,875 DEBUG [Thread-1140 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:41:33,875 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:33,875 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:33,875 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:41:33,875 INFO [Thread-1134 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1134. 2024-11-23T15:41:33,875 DEBUG [Thread-1134 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:41:33,875 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:33,875 DEBUG [Thread-1134 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:33,875 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:41:33,876 INFO [Thread-1142 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1142. 2024-11-23T15:41:33,876 DEBUG [Thread-1142 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:41:33,876 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:33,876 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:33,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:41:33,877 INFO [Thread-1136 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1136. 2024-11-23T15:41:33,877 DEBUG [Thread-1136 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:41:33,877 INFO [Thread-1138 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1138. 2024-11-23T15:41:33,877 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:33,877 DEBUG [Thread-1138 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:41:33,877 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:33,877 DEBUG [Thread-1136 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:33,878 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:41:33,878 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:33,878 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:41:33,892 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/302fe0b0d9d94177aed9e95e4da03d03 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/302fe0b0d9d94177aed9e95e4da03d03 2024-11-23T15:41:33,893 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/23b5fbe576eb45ff9a90ab6636a68c8d as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/23b5fbe576eb45ff9a90ab6636a68c8d 2024-11-23T15:41:33,898 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/B of d2607dde3b91c49826fdae09a26d7b59 into 302fe0b0d9d94177aed9e95e4da03d03(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:33,898 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:33,898 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/B, priority=13, startTime=1732376493465; duration=0sec 2024-11-23T15:41:33,898 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/A of d2607dde3b91c49826fdae09a26d7b59 into 23b5fbe576eb45ff9a90ab6636a68c8d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:33,898 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:33,898 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:33,898 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:B 2024-11-23T15:41:33,898 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/A, priority=13, startTime=1732376493465; duration=0sec 2024-11-23T15:41:33,898 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:33,898 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:33,898 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:A 2024-11-23T15:41:33,899 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:33,899 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): d2607dde3b91c49826fdae09a26d7b59/C is initiating minor compaction (all files) 2024-11-23T15:41:33,899 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d2607dde3b91c49826fdae09a26d7b59/C in TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:33,899 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/50e6628e28f04a839fd7f83e547f3b8c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/59597fda80eb4417a1825419fb3d5de0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/aba21285a82241a884c374e3c7d3aea9] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp, totalSize=36.9 K 2024-11-23T15:41:33,900 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 50e6628e28f04a839fd7f83e547f3b8c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1732376487099 2024-11-23T15:41:33,900 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 59597fda80eb4417a1825419fb3d5de0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732376489387 2024-11-23T15:41:33,900 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting aba21285a82241a884c374e3c7d3aea9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1732376491557 2024-11-23T15:41:33,907 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d2607dde3b91c49826fdae09a26d7b59#C#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:33,907 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/c689f4ff37934ddf8c1fbc7782d7e83e is 50, key is test_row_0/C:col10/1732376491585/Put/seqid=0 2024-11-23T15:41:33,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742159_1335 (size=13323) 2024-11-23T15:41:34,039 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/a47e9182d0214a538c386beb5efe6dc8 2024-11-23T15:41:34,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/bb3da8dfabdf43dfb209b4a180bb8bd3 is 50, key is test_row_0/B:col10/1732376492236/Put/seqid=0 2024-11-23T15:41:34,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742160_1336 (size=12301) 2024-11-23T15:41:34,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:34,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. as already flushing 2024-11-23T15:41:34,176 INFO [Thread-1127 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1127. 2024-11-23T15:41:34,176 DEBUG [Thread-1127 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:41:34,176 DEBUG [Thread-1127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:34,176 DEBUG [Thread-1127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:34,176 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:41:34,322 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/c689f4ff37934ddf8c1fbc7782d7e83e as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/c689f4ff37934ddf8c1fbc7782d7e83e 2024-11-23T15:41:34,330 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d2607dde3b91c49826fdae09a26d7b59/C of d2607dde3b91c49826fdae09a26d7b59 into c689f4ff37934ddf8c1fbc7782d7e83e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:34,330 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:34,330 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., storeName=d2607dde3b91c49826fdae09a26d7b59/C, priority=13, startTime=1732376493466; duration=0sec 2024-11-23T15:41:34,331 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:34,331 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d2607dde3b91c49826fdae09a26d7b59:C 2024-11-23T15:41:34,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-23T15:41:34,389 INFO [Thread-1131 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1131. 2024-11-23T15:41:34,389 DEBUG [Thread-1131 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:41:34,389 DEBUG [Thread-1131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:34,389 INFO [Thread-1123 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1123. 2024-11-23T15:41:34,389 DEBUG [Thread-1131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:34,389 DEBUG [Thread-1123 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:41:34,389 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:34,389 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:41:34,390 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:34,390 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:41:34,396 INFO [Thread-1129 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1129. 2024-11-23T15:41:34,396 INFO [Thread-1125 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1125. 2024-11-23T15:41:34,396 DEBUG [Thread-1129 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:41:34,396 DEBUG [Thread-1125 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:41:34,396 DEBUG [Thread-1129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:34,396 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:34,396 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:34,396 DEBUG [Thread-1129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:34,396 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:41:34,396 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:41:34,452 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/bb3da8dfabdf43dfb209b4a180bb8bd3 2024-11-23T15:41:34,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/a3e7be62216c4c64b8adf98695f0b886 is 50, key is test_row_0/C:col10/1732376492236/Put/seqid=0 2024-11-23T15:41:34,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742161_1337 (size=12301) 2024-11-23T15:41:34,870 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/a3e7be62216c4c64b8adf98695f0b886 2024-11-23T15:41:34,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/a47e9182d0214a538c386beb5efe6dc8 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/a47e9182d0214a538c386beb5efe6dc8 2024-11-23T15:41:34,884 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/a47e9182d0214a538c386beb5efe6dc8, entries=150, sequenceid=449, filesize=12.0 K 2024-11-23T15:41:34,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/bb3da8dfabdf43dfb209b4a180bb8bd3 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bb3da8dfabdf43dfb209b4a180bb8bd3 2024-11-23T15:41:34,889 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bb3da8dfabdf43dfb209b4a180bb8bd3, entries=150, sequenceid=449, filesize=12.0 K 2024-11-23T15:41:34,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/a3e7be62216c4c64b8adf98695f0b886 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/a3e7be62216c4c64b8adf98695f0b886 2024-11-23T15:41:34,894 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/a3e7be62216c4c64b8adf98695f0b886, entries=150, sequenceid=449, filesize=12.0 K 2024-11-23T15:41:34,894 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=33.54 KB/34350 for d2607dde3b91c49826fdae09a26d7b59 in 1283ms, sequenceid=449, compaction requested=false 2024-11-23T15:41:34,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for d2607dde3b91c49826fdae09a26d7b59: 2024-11-23T15:41:34,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:34,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-23T15:41:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-23T15:41:34,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-23T15:41:34,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6590 sec 2024-11-23T15:41:34,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 2.6630 sec 2024-11-23T15:41:36,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-23T15:41:36,405 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:36,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T15:41:36,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-23T15:41:36,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-23T15:41:36,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-23T15:41:36,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-23T15:41:36,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-23T15:41:36,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T15:41:36,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3704 2024-11-23T15:41:36,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3725 2024-11-23T15:41:36,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3643 2024-11-23T15:41:36,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3688 2024-11-23T15:41:36,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3745 2024-11-23T15:41:36,405 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T15:41:36,405 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T15:41:36,405 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T15:41:36,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:36,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:36,406 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:41:36,407 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-23T15:41:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T15:41:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-23T15:41:36,412 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376496411"}]},"ts":"1732376496411"} 2024-11-23T15:41:36,414 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T15:41:36,414 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T15:41:36,415 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T15:41:36,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2607dde3b91c49826fdae09a26d7b59, UNASSIGN}] 2024-11-23T15:41:36,418 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2607dde3b91c49826fdae09a26d7b59, UNASSIGN 2024-11-23T15:41:36,419 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=d2607dde3b91c49826fdae09a26d7b59, regionState=CLOSING, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:36,421 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2607dde3b91c49826fdae09a26d7b59, UNASSIGN because future has completed 2024-11-23T15:41:36,421 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T15:41:36,421 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194}] 2024-11-23T15:41:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-23T15:41:36,574 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:36,575 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T15:41:36,575 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing d2607dde3b91c49826fdae09a26d7b59, disabling compactions & flushes 2024-11-23T15:41:36,575 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:36,575 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:36,575 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. after waiting 0 ms 2024-11-23T15:41:36,575 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:36,575 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(2902): Flushing d2607dde3b91c49826fdae09a26d7b59 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-23T15:41:36,584 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/863e337308dc44d285727fedc0e7109e is 50, key is test_row_0/A:col10/1732376494387/Put/seqid=0 2024-11-23T15:41:36,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742162_1338 (size=9857) 2024-11-23T15:41:36,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-23T15:41:36,989 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/863e337308dc44d285727fedc0e7109e 2024-11-23T15:41:37,003 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/0ac860050df147b282667a1ed62df642 is 50, key is test_row_0/B:col10/1732376494387/Put/seqid=0 2024-11-23T15:41:37,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742163_1339 (size=9857) 2024-11-23T15:41:37,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-23T15:41:37,410 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/0ac860050df147b282667a1ed62df642 2024-11-23T15:41:37,424 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/89c2616ee096488cbd082cc065b9b7ed is 50, key is test_row_0/C:col10/1732376494387/Put/seqid=0 2024-11-23T15:41:37,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742164_1340 (size=9857) 2024-11-23T15:41:37,451 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T15:41:37,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-23T15:41:37,832 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/89c2616ee096488cbd082cc065b9b7ed 2024-11-23T15:41:37,842 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/A/863e337308dc44d285727fedc0e7109e as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/863e337308dc44d285727fedc0e7109e 2024-11-23T15:41:37,847 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/863e337308dc44d285727fedc0e7109e, entries=100, sequenceid=460, filesize=9.6 K 2024-11-23T15:41:37,848 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/B/0ac860050df147b282667a1ed62df642 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/0ac860050df147b282667a1ed62df642 2024-11-23T15:41:37,852 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/0ac860050df147b282667a1ed62df642, entries=100, sequenceid=460, filesize=9.6 K 2024-11-23T15:41:37,853 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/.tmp/C/89c2616ee096488cbd082cc065b9b7ed as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/89c2616ee096488cbd082cc065b9b7ed 2024-11-23T15:41:37,858 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/89c2616ee096488cbd082cc065b9b7ed, entries=100, sequenceid=460, filesize=9.6 K 2024-11-23T15:41:37,859 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for d2607dde3b91c49826fdae09a26d7b59 in 1283ms, sequenceid=460, compaction requested=true 2024-11-23T15:41:37,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/638edb96809d4f9fa851a1be1b0b5b72, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/2bfebe086d81466eb50ef42306993d45, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6d67efc44a1b4f89873d762032dde51e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/ced9086b11c74206bc07d4755d841e8e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/1744e2a29781470b8fd0988bc34518c9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/e91e1b8438004b62a195ebdc217ca89a, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6e08e44f115d4723968963a20db33eb8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/52c4ba5afb9c42ca81bd1a4ada310b89, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/901a58b0bc9d47a691a3477c84ba9bcb, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/ab3c6a691117411fa9302f246f02e9f8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/9b17080f80534e709009559586e6b751, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/b4456b929f324bc2b028e8a951521db7, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/58ae63165a3e4e00b4e637038051e21f, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/657154ca44cc49528f1749fbcc898c02, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/9bf8e924f6394291b77d97a87ededb98, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/ec33126b2b1e403e8d9f9c438292b96f, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/155f3979688c430b9a3ad5daae62d391, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/1d9934fe18e94344904a99a3e607b6fb, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6527a08773e243bba2eb145b98a0f11d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/7607ab7c35d6405cac456d5224acfd43, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/770e7d31d8834b79adc940d400ed28f2, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/b36e35cbbbc54844b4d2d3b470e3fd59, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/fb603ef9af57484cb2157c7600ddb9d7, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/3820563835304856a3ec866bdaeadcfc, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/d1739b2dcba14483b9e22b5d777fcbec, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/0bd9f8d861814843881a081b256da168, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/28c52e7a8c294ef4b5d640437752f930, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/1cb851247d1d414994eec6b001d9fbf0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/34d159b362ae4c32b616855049913011, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6a174b6c6c734bbdbd41c6b0ab72ae33] to archive 2024-11-23T15:41:37,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T15:41:37,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/638edb96809d4f9fa851a1be1b0b5b72 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/638edb96809d4f9fa851a1be1b0b5b72 2024-11-23T15:41:37,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/2bfebe086d81466eb50ef42306993d45 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/2bfebe086d81466eb50ef42306993d45 2024-11-23T15:41:37,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6d67efc44a1b4f89873d762032dde51e to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6d67efc44a1b4f89873d762032dde51e 2024-11-23T15:41:37,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/ced9086b11c74206bc07d4755d841e8e to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/ced9086b11c74206bc07d4755d841e8e 2024-11-23T15:41:37,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/1744e2a29781470b8fd0988bc34518c9 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/1744e2a29781470b8fd0988bc34518c9 2024-11-23T15:41:37,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/e91e1b8438004b62a195ebdc217ca89a to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/e91e1b8438004b62a195ebdc217ca89a 2024-11-23T15:41:37,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6e08e44f115d4723968963a20db33eb8 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6e08e44f115d4723968963a20db33eb8 2024-11-23T15:41:37,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/52c4ba5afb9c42ca81bd1a4ada310b89 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/52c4ba5afb9c42ca81bd1a4ada310b89 2024-11-23T15:41:37,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/901a58b0bc9d47a691a3477c84ba9bcb to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/901a58b0bc9d47a691a3477c84ba9bcb 2024-11-23T15:41:37,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/ab3c6a691117411fa9302f246f02e9f8 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/ab3c6a691117411fa9302f246f02e9f8 2024-11-23T15:41:37,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/9b17080f80534e709009559586e6b751 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/9b17080f80534e709009559586e6b751 2024-11-23T15:41:37,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/b4456b929f324bc2b028e8a951521db7 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/b4456b929f324bc2b028e8a951521db7 2024-11-23T15:41:37,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/58ae63165a3e4e00b4e637038051e21f to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/58ae63165a3e4e00b4e637038051e21f 2024-11-23T15:41:37,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/657154ca44cc49528f1749fbcc898c02 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/657154ca44cc49528f1749fbcc898c02 2024-11-23T15:41:37,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/9bf8e924f6394291b77d97a87ededb98 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/9bf8e924f6394291b77d97a87ededb98 2024-11-23T15:41:37,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/ec33126b2b1e403e8d9f9c438292b96f to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/ec33126b2b1e403e8d9f9c438292b96f 2024-11-23T15:41:37,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/155f3979688c430b9a3ad5daae62d391 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/155f3979688c430b9a3ad5daae62d391 2024-11-23T15:41:37,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/1d9934fe18e94344904a99a3e607b6fb to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/1d9934fe18e94344904a99a3e607b6fb 2024-11-23T15:41:37,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6527a08773e243bba2eb145b98a0f11d to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6527a08773e243bba2eb145b98a0f11d 2024-11-23T15:41:37,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/7607ab7c35d6405cac456d5224acfd43 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/7607ab7c35d6405cac456d5224acfd43 2024-11-23T15:41:37,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/770e7d31d8834b79adc940d400ed28f2 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/770e7d31d8834b79adc940d400ed28f2 2024-11-23T15:41:37,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/b36e35cbbbc54844b4d2d3b470e3fd59 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/b36e35cbbbc54844b4d2d3b470e3fd59 2024-11-23T15:41:37,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/fb603ef9af57484cb2157c7600ddb9d7 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/fb603ef9af57484cb2157c7600ddb9d7 2024-11-23T15:41:37,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/3820563835304856a3ec866bdaeadcfc to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/3820563835304856a3ec866bdaeadcfc 2024-11-23T15:41:37,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/d1739b2dcba14483b9e22b5d777fcbec to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/d1739b2dcba14483b9e22b5d777fcbec 2024-11-23T15:41:37,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/0bd9f8d861814843881a081b256da168 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/0bd9f8d861814843881a081b256da168 2024-11-23T15:41:37,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/28c52e7a8c294ef4b5d640437752f930 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/28c52e7a8c294ef4b5d640437752f930 2024-11-23T15:41:37,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/1cb851247d1d414994eec6b001d9fbf0 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/1cb851247d1d414994eec6b001d9fbf0 2024-11-23T15:41:37,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/34d159b362ae4c32b616855049913011 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/34d159b362ae4c32b616855049913011 2024-11-23T15:41:37,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6a174b6c6c734bbdbd41c6b0ab72ae33 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/6a174b6c6c734bbdbd41c6b0ab72ae33 2024-11-23T15:41:37,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/4686e0b4d0ab480db7fa7dbec61a48ae, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/8e54b1561d67472c9287b6ecc1c22c20, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/4527cb53b13b4612bd8a7c5c1a0eb5e9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/6d945cd1d47046b59836c71759a50357, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/6677d10f060645e1bb4681cf14166348, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/9a68f13fa1944297a89f1c0fb92f065d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/3896dc1347ea4f9da0ea16998e4184c5, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/9f1eb00a8c754d2c97e0e9c8b36b2272, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/1199f9b293de47caa24e8d67ccc061f5, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/36ed39207174475985821bd619051ea6, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/a61089bd13da4910ae89088cb896c468, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/d6a26c78c00c4e44ab7060937d990971, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/a260bb45be1445e09f475d9f19951e57, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/7675c227ee95421d88030a96e0e2344b, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/b45020ca5a734331a039fbca3b1aff04, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/8591ce6f3ba7420c9810db111690d3be, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/6bacd0cda6d84501ac99f8d165cbaf41, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/db2c8804121144daa780d81c0646477e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/63f927ab625a48e98873e853b0cbe0c6, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/3011f8cc32da4eb79796c5c5c3b18df2, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/fc0d965933594b05b2764a89d208328e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/3acd34999566408098c9b4bf9ca9705e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bfe94dfb623e4170af285849a85bc532, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bf5a6ffe6e1848c2a0e1fd76015f8d9b, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bb1da1c161fd4d49a63fbd7fcd7a6ced, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/ee18f23db8eb4eb6aafe2e4351177541, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/1388c296dc0c41609dd4a9b8175b44fb, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/f0ccf2f9731a44e0a04b67f1a526d200, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/babb3189270b48b98b71a23e33f3e925, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bad58d697fae4e6f943bb8be623289dc] to archive 2024-11-23T15:41:37,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T15:41:37,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/4686e0b4d0ab480db7fa7dbec61a48ae to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/4686e0b4d0ab480db7fa7dbec61a48ae 2024-11-23T15:41:37,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/8e54b1561d67472c9287b6ecc1c22c20 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/8e54b1561d67472c9287b6ecc1c22c20 2024-11-23T15:41:37,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/4527cb53b13b4612bd8a7c5c1a0eb5e9 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/4527cb53b13b4612bd8a7c5c1a0eb5e9 2024-11-23T15:41:37,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/6d945cd1d47046b59836c71759a50357 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/6d945cd1d47046b59836c71759a50357 2024-11-23T15:41:37,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/6677d10f060645e1bb4681cf14166348 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/6677d10f060645e1bb4681cf14166348 2024-11-23T15:41:37,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/9a68f13fa1944297a89f1c0fb92f065d to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/9a68f13fa1944297a89f1c0fb92f065d 2024-11-23T15:41:37,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/3896dc1347ea4f9da0ea16998e4184c5 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/3896dc1347ea4f9da0ea16998e4184c5 2024-11-23T15:41:37,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/9f1eb00a8c754d2c97e0e9c8b36b2272 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/9f1eb00a8c754d2c97e0e9c8b36b2272 2024-11-23T15:41:37,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/1199f9b293de47caa24e8d67ccc061f5 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/1199f9b293de47caa24e8d67ccc061f5 2024-11-23T15:41:37,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/36ed39207174475985821bd619051ea6 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/36ed39207174475985821bd619051ea6 2024-11-23T15:41:37,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/a61089bd13da4910ae89088cb896c468 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/a61089bd13da4910ae89088cb896c468 2024-11-23T15:41:37,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/d6a26c78c00c4e44ab7060937d990971 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/d6a26c78c00c4e44ab7060937d990971 2024-11-23T15:41:37,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/a260bb45be1445e09f475d9f19951e57 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/a260bb45be1445e09f475d9f19951e57 2024-11-23T15:41:37,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/7675c227ee95421d88030a96e0e2344b to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/7675c227ee95421d88030a96e0e2344b 2024-11-23T15:41:37,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/b45020ca5a734331a039fbca3b1aff04 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/b45020ca5a734331a039fbca3b1aff04 2024-11-23T15:41:37,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/8591ce6f3ba7420c9810db111690d3be to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/8591ce6f3ba7420c9810db111690d3be 2024-11-23T15:41:37,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/6bacd0cda6d84501ac99f8d165cbaf41 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/6bacd0cda6d84501ac99f8d165cbaf41 2024-11-23T15:41:37,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/db2c8804121144daa780d81c0646477e to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/db2c8804121144daa780d81c0646477e 2024-11-23T15:41:37,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/63f927ab625a48e98873e853b0cbe0c6 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/63f927ab625a48e98873e853b0cbe0c6 2024-11-23T15:41:37,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/3011f8cc32da4eb79796c5c5c3b18df2 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/3011f8cc32da4eb79796c5c5c3b18df2 2024-11-23T15:41:37,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/fc0d965933594b05b2764a89d208328e to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/fc0d965933594b05b2764a89d208328e 2024-11-23T15:41:37,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/3acd34999566408098c9b4bf9ca9705e to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/3acd34999566408098c9b4bf9ca9705e 2024-11-23T15:41:37,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bfe94dfb623e4170af285849a85bc532 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bfe94dfb623e4170af285849a85bc532 2024-11-23T15:41:37,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bf5a6ffe6e1848c2a0e1fd76015f8d9b to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bf5a6ffe6e1848c2a0e1fd76015f8d9b 2024-11-23T15:41:37,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bb1da1c161fd4d49a63fbd7fcd7a6ced to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bb1da1c161fd4d49a63fbd7fcd7a6ced 2024-11-23T15:41:37,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/ee18f23db8eb4eb6aafe2e4351177541 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/ee18f23db8eb4eb6aafe2e4351177541 2024-11-23T15:41:37,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/1388c296dc0c41609dd4a9b8175b44fb to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/1388c296dc0c41609dd4a9b8175b44fb 2024-11-23T15:41:37,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/f0ccf2f9731a44e0a04b67f1a526d200 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/f0ccf2f9731a44e0a04b67f1a526d200 2024-11-23T15:41:37,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/babb3189270b48b98b71a23e33f3e925 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/babb3189270b48b98b71a23e33f3e925 2024-11-23T15:41:37,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bad58d697fae4e6f943bb8be623289dc to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bad58d697fae4e6f943bb8be623289dc 2024-11-23T15:41:37,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/acf92422bfc249d7b829e0b1cc384619, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/39e4069f94114d238b039f912deb3945, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/c739ffb6532c4162825f5d2ad23147ba, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/6c55da66c480454287c5f44f554ab712, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/6117a615fb2747129c014c974a396dbf, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/230d133064584d3489b62add060d6579, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/6bd128f800f74ee5825c61ba5267cd59, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/0a545a3024004e2a93077eb565f88db1, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/f806cea6d94148129b384f99215d3d7d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/0e1f5284df964499b0dfad3c3c7c9cfd, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/665f0d06503543bb9d5b3875e9741f90, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/4ae1dd7445b84618885ff4882f73b37b, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/5124158e765e410ba999d1af8ad81bbe, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/aafe0c70717844f0a64a3e3ca5516a17, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/2a927173e1994663bacf53d5187fab1d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/f997deec88ea47b8841afb9af47d2a2e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/678fefc1b6f04ebda48c27be509e5b67, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/51633d0a66f142829667f5b9847211fc, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/cdaf1d7093474be4b21ce3a2d9e307a3, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/b950c225012a4379a9a7f811a1a3e045, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/54586241b63d43f9afcb8e62a6575d55, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/b3cc66987b5c404e8f5a7bf488c60054, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/5c904bc1363a4294bb09fa0e18a628cf, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/22d60596680d4ed999485d9bb806c7fd, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/dbd492977d62471b9ba004acb92c9e83, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/b5ec23e4db3f4740ada1c54aa95d13c9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/50e6628e28f04a839fd7f83e547f3b8c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/729249a42fa143a59582d5cbf8a6396c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/59597fda80eb4417a1825419fb3d5de0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/aba21285a82241a884c374e3c7d3aea9] to archive 2024-11-23T15:41:37,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T15:41:37,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/acf92422bfc249d7b829e0b1cc384619 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/acf92422bfc249d7b829e0b1cc384619 2024-11-23T15:41:37,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/39e4069f94114d238b039f912deb3945 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/39e4069f94114d238b039f912deb3945 2024-11-23T15:41:37,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/c739ffb6532c4162825f5d2ad23147ba to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/c739ffb6532c4162825f5d2ad23147ba 2024-11-23T15:41:37,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/6c55da66c480454287c5f44f554ab712 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/6c55da66c480454287c5f44f554ab712 2024-11-23T15:41:37,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/6117a615fb2747129c014c974a396dbf to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/6117a615fb2747129c014c974a396dbf 2024-11-23T15:41:37,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/230d133064584d3489b62add060d6579 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/230d133064584d3489b62add060d6579 2024-11-23T15:41:37,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/6bd128f800f74ee5825c61ba5267cd59 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/6bd128f800f74ee5825c61ba5267cd59 2024-11-23T15:41:37,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/0a545a3024004e2a93077eb565f88db1 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/0a545a3024004e2a93077eb565f88db1 2024-11-23T15:41:37,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/f806cea6d94148129b384f99215d3d7d to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/f806cea6d94148129b384f99215d3d7d 2024-11-23T15:41:37,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/0e1f5284df964499b0dfad3c3c7c9cfd to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/0e1f5284df964499b0dfad3c3c7c9cfd 2024-11-23T15:41:37,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/665f0d06503543bb9d5b3875e9741f90 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/665f0d06503543bb9d5b3875e9741f90 2024-11-23T15:41:37,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/4ae1dd7445b84618885ff4882f73b37b to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/4ae1dd7445b84618885ff4882f73b37b 2024-11-23T15:41:37,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/5124158e765e410ba999d1af8ad81bbe to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/5124158e765e410ba999d1af8ad81bbe 2024-11-23T15:41:37,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/aafe0c70717844f0a64a3e3ca5516a17 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/aafe0c70717844f0a64a3e3ca5516a17 2024-11-23T15:41:37,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/2a927173e1994663bacf53d5187fab1d to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/2a927173e1994663bacf53d5187fab1d 2024-11-23T15:41:37,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/f997deec88ea47b8841afb9af47d2a2e to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/f997deec88ea47b8841afb9af47d2a2e 2024-11-23T15:41:37,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/678fefc1b6f04ebda48c27be509e5b67 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/678fefc1b6f04ebda48c27be509e5b67 2024-11-23T15:41:37,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/51633d0a66f142829667f5b9847211fc to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/51633d0a66f142829667f5b9847211fc 2024-11-23T15:41:37,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/cdaf1d7093474be4b21ce3a2d9e307a3 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/cdaf1d7093474be4b21ce3a2d9e307a3 2024-11-23T15:41:37,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/b950c225012a4379a9a7f811a1a3e045 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/b950c225012a4379a9a7f811a1a3e045 2024-11-23T15:41:37,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/54586241b63d43f9afcb8e62a6575d55 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/54586241b63d43f9afcb8e62a6575d55 2024-11-23T15:41:37,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/b3cc66987b5c404e8f5a7bf488c60054 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/b3cc66987b5c404e8f5a7bf488c60054 2024-11-23T15:41:37,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/5c904bc1363a4294bb09fa0e18a628cf to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/5c904bc1363a4294bb09fa0e18a628cf 2024-11-23T15:41:37,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/22d60596680d4ed999485d9bb806c7fd to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/22d60596680d4ed999485d9bb806c7fd 2024-11-23T15:41:37,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/dbd492977d62471b9ba004acb92c9e83 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/dbd492977d62471b9ba004acb92c9e83 2024-11-23T15:41:37,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/b5ec23e4db3f4740ada1c54aa95d13c9 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/b5ec23e4db3f4740ada1c54aa95d13c9 2024-11-23T15:41:37,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/50e6628e28f04a839fd7f83e547f3b8c to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/50e6628e28f04a839fd7f83e547f3b8c 2024-11-23T15:41:37,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/729249a42fa143a59582d5cbf8a6396c to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/729249a42fa143a59582d5cbf8a6396c 2024-11-23T15:41:37,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/59597fda80eb4417a1825419fb3d5de0 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/59597fda80eb4417a1825419fb3d5de0 2024-11-23T15:41:37,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/aba21285a82241a884c374e3c7d3aea9 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/aba21285a82241a884c374e3c7d3aea9 2024-11-23T15:41:37,945 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/recovered.edits/463.seqid, newMaxSeqId=463, maxSeqId=1 2024-11-23T15:41:37,945 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. 2024-11-23T15:41:37,946 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for d2607dde3b91c49826fdae09a26d7b59: Waiting for close lock at 1732376496575Running coprocessor pre-close hooks at 1732376496575Disabling compacts and flushes for region at 1732376496575Disabling writes for close at 1732376496575Obtaining lock to block concurrent updates at 1732376496575Preparing flush snapshotting stores in d2607dde3b91c49826fdae09a26d7b59 at 1732376496575Finished memstore snapshotting TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732376496576 (+1 ms)Flushing stores of TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59. at 1732376496577 (+1 ms)Flushing d2607dde3b91c49826fdae09a26d7b59/A: creating writer at 1732376496577Flushing d2607dde3b91c49826fdae09a26d7b59/A: appending metadata at 1732376496583 (+6 ms)Flushing d2607dde3b91c49826fdae09a26d7b59/A: closing flushed file at 1732376496583Flushing d2607dde3b91c49826fdae09a26d7b59/B: creating writer at 1732376497000 (+417 ms)Flushing d2607dde3b91c49826fdae09a26d7b59/B: appending metadata at 1732376497002 (+2 ms)Flushing d2607dde3b91c49826fdae09a26d7b59/B: closing flushed file at 1732376497002Flushing d2607dde3b91c49826fdae09a26d7b59/C: creating writer at 1732376497419 (+417 ms)Flushing d2607dde3b91c49826fdae09a26d7b59/C: appending metadata at 1732376497423 (+4 ms)Flushing d2607dde3b91c49826fdae09a26d7b59/C: closing flushed file at 1732376497423Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d2ace10: reopening flushed file at 1732376497841 (+418 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ad1119: reopening flushed file at 1732376497847 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@cb947d5: reopening flushed file at 1732376497852 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for d2607dde3b91c49826fdae09a26d7b59 in 1283ms, sequenceid=460, compaction requested=true at 1732376497859 (+7 ms)Writing region close event to WAL at 1732376497943 (+84 ms)Running coprocessor post-close hooks at 1732376497945 (+2 ms)Closed at 1732376497945 2024-11-23T15:41:37,947 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:37,948 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=d2607dde3b91c49826fdae09a26d7b59, regionState=CLOSED 2024-11-23T15:41:37,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 because future has completed 2024-11-23T15:41:37,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-23T15:41:37,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure d2607dde3b91c49826fdae09a26d7b59, server=b712f9af2c12,46589,1732376409194 in 1.5280 sec 2024-11-23T15:41:37,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-23T15:41:37,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d2607dde3b91c49826fdae09a26d7b59, UNASSIGN in 1.5330 sec 2024-11-23T15:41:37,953 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-23T15:41:37,953 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5370 sec 2024-11-23T15:41:37,954 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376497954"}]},"ts":"1732376497954"} 2024-11-23T15:41:37,955 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T15:41:37,955 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T15:41:37,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5480 sec 2024-11-23T15:41:38,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-23T15:41:38,565 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:38,567 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-23T15:41:38,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:41:38,571 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=94, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:41:38,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-23T15:41:38,573 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=94, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:41:38,576 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:38,580 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A, FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B, FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C, FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/recovered.edits] 2024-11-23T15:41:38,584 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/23b5fbe576eb45ff9a90ab6636a68c8d to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/23b5fbe576eb45ff9a90ab6636a68c8d 2024-11-23T15:41:38,586 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/863e337308dc44d285727fedc0e7109e to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/863e337308dc44d285727fedc0e7109e 2024-11-23T15:41:38,588 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/a47e9182d0214a538c386beb5efe6dc8 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/A/a47e9182d0214a538c386beb5efe6dc8 2024-11-23T15:41:38,592 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/0ac860050df147b282667a1ed62df642 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/0ac860050df147b282667a1ed62df642 2024-11-23T15:41:38,594 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/302fe0b0d9d94177aed9e95e4da03d03 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/302fe0b0d9d94177aed9e95e4da03d03 2024-11-23T15:41:38,596 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bb3da8dfabdf43dfb209b4a180bb8bd3 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/B/bb3da8dfabdf43dfb209b4a180bb8bd3 2024-11-23T15:41:38,600 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/89c2616ee096488cbd082cc065b9b7ed to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/89c2616ee096488cbd082cc065b9b7ed 2024-11-23T15:41:38,602 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/a3e7be62216c4c64b8adf98695f0b886 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/a3e7be62216c4c64b8adf98695f0b886 2024-11-23T15:41:38,604 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/c689f4ff37934ddf8c1fbc7782d7e83e to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/C/c689f4ff37934ddf8c1fbc7782d7e83e 2024-11-23T15:41:38,607 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/recovered.edits/463.seqid to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59/recovered.edits/463.seqid 2024-11-23T15:41:38,607 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/d2607dde3b91c49826fdae09a26d7b59 2024-11-23T15:41:38,608 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T15:41:38,609 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=94, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:41:38,613 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T15:41:38,615 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T15:41:38,633 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=94, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:41:38,633 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T15:41:38,633 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732376498633"}]},"ts":"9223372036854775807"} 2024-11-23T15:41:38,635 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T15:41:38,635 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d2607dde3b91c49826fdae09a26d7b59, NAME => 'TestAcidGuarantees,,1732376472670.d2607dde3b91c49826fdae09a26d7b59.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T15:41:38,635 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T15:41:38,636 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732376498635"}]},"ts":"9223372036854775807"} 2024-11-23T15:41:38,637 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T15:41:38,638 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=94, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:41:38,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 71 msec 2024-11-23T15:41:38,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-23T15:41:38,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T15:41:38,684 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:38,693 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 242), OpenFileDescriptor=453 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=478 (was 417) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7715 (was 7222) - AvailableMemoryMB LEAK? - 2024-11-23T15:41:38,701 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=478, ProcessCount=11, AvailableMemoryMB=7715 2024-11-23T15:41:38,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T15:41:38,702 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:41:38,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T15:41:38,704 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T15:41:38,704 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:38,704 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 95 2024-11-23T15:41:38,705 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T15:41:38,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-23T15:41:38,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742165_1341 (size=959) 2024-11-23T15:41:38,768 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-23T15:41:38,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-23T15:41:39,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-23T15:41:39,115 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e 2024-11-23T15:41:39,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742166_1342 (size=53) 2024-11-23T15:41:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-23T15:41:39,525 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:41:39,525 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6680d3b6fc70270ea7a87351f7da4298, disabling compactions & flushes 2024-11-23T15:41:39,525 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:39,525 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:39,525 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. after waiting 0 ms 2024-11-23T15:41:39,525 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:39,525 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:39,525 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6680d3b6fc70270ea7a87351f7da4298: Waiting for close lock at 1732376499525Disabling compacts and flushes for region at 1732376499525Disabling writes for close at 1732376499525Writing region close event to WAL at 1732376499525Closed at 1732376499525 2024-11-23T15:41:39,528 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T15:41:39,528 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732376499528"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732376499528"}]},"ts":"1732376499528"} 2024-11-23T15:41:39,532 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T15:41:39,533 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T15:41:39,534 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376499533"}]},"ts":"1732376499533"} 2024-11-23T15:41:39,535 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T15:41:39,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6680d3b6fc70270ea7a87351f7da4298, ASSIGN}] 2024-11-23T15:41:39,536 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6680d3b6fc70270ea7a87351f7da4298, ASSIGN 2024-11-23T15:41:39,537 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6680d3b6fc70270ea7a87351f7da4298, ASSIGN; state=OFFLINE, location=b712f9af2c12,46589,1732376409194; forceNewPlan=false, retain=false 2024-11-23T15:41:39,688 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=6680d3b6fc70270ea7a87351f7da4298, regionState=OPENING, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:39,691 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6680d3b6fc70270ea7a87351f7da4298, ASSIGN because future has completed 2024-11-23T15:41:39,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194}] 2024-11-23T15:41:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-23T15:41:39,848 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:39,848 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7752): Opening region: {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:41:39,849 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:39,849 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:41:39,849 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7794): checking encryption for 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:39,849 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(7797): checking classloading for 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:39,850 INFO [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:39,852 INFO [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6680d3b6fc70270ea7a87351f7da4298 columnFamilyName A 2024-11-23T15:41:39,852 DEBUG [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:39,853 INFO [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] regionserver.HStore(327): Store=6680d3b6fc70270ea7a87351f7da4298/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:41:39,853 INFO [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:39,854 INFO [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6680d3b6fc70270ea7a87351f7da4298 columnFamilyName B 2024-11-23T15:41:39,854 DEBUG [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:39,855 INFO [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] regionserver.HStore(327): Store=6680d3b6fc70270ea7a87351f7da4298/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:41:39,855 INFO [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:39,857 INFO [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6680d3b6fc70270ea7a87351f7da4298 columnFamilyName C 2024-11-23T15:41:39,857 DEBUG [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:39,857 INFO [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] regionserver.HStore(327): Store=6680d3b6fc70270ea7a87351f7da4298/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:41:39,857 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1038): replaying wal for 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:39,858 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:39,859 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:39,860 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1048): stopping wal replay for 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:39,860 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1060): Cleaning up temporary data for 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:39,861 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T15:41:39,863 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1093): writing seq id for 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:39,866 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:41:39,866 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1114): Opened 6680d3b6fc70270ea7a87351f7da4298; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63404628, jitterRate=-0.05519741773605347}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T15:41:39,866 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:39,867 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegion(1006): Region open journal for 6680d3b6fc70270ea7a87351f7da4298: Running coprocessor pre-open hook at 1732376499849Writing region info on filesystem at 1732376499849Initializing all the Stores at 1732376499850 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376499850Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376499850Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376499850Cleaning up temporary data from old regions at 1732376499860 (+10 ms)Running coprocessor post-open hooks at 1732376499866 (+6 ms)Region opened successfully at 1732376499867 (+1 ms) 2024-11-23T15:41:39,868 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., pid=97, masterSystemTime=1732376499845 2024-11-23T15:41:39,871 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:39,871 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=97}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:39,872 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=6680d3b6fc70270ea7a87351f7da4298, regionState=OPEN, openSeqNum=2, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:39,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 because future has completed 2024-11-23T15:41:39,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-23T15:41:39,878 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure 6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 in 184 msec 2024-11-23T15:41:39,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-23T15:41:39,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6680d3b6fc70270ea7a87351f7da4298, ASSIGN in 343 msec 2024-11-23T15:41:39,882 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T15:41:39,882 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376499882"}]},"ts":"1732376499882"} 2024-11-23T15:41:39,884 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T15:41:39,885 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=95, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T15:41:39,887 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1830 sec 2024-11-23T15:41:40,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-23T15:41:40,856 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:40,858 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7befe57f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:40,858 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:41:40,858 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:41:40,861 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:41:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:41:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:41:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c815554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:41:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:41:40,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:40,863 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:41:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55ed8acf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:40,864 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:41:40,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:41:40,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:41:40,867 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:41:40,867 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T15:41:40,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is b712f9af2c12,42351,1732376408377 2024-11-23T15:41:40,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7b49d0e3 2024-11-23T15:41:40,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T15:41:40,869 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T15:41:40,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T15:41:40,871 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:41:40,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=98, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-23T15:41:40,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742167_1343 (size=995) 2024-11-23T15:41:41,291 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-23T15:41:41,292 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-23T15:41:41,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T15:41:41,300 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6680d3b6fc70270ea7a87351f7da4298, REOPEN/MOVE}] 2024-11-23T15:41:41,300 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6680d3b6fc70270ea7a87351f7da4298, REOPEN/MOVE 2024-11-23T15:41:41,301 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=6680d3b6fc70270ea7a87351f7da4298, regionState=CLOSING, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:41,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6680d3b6fc70270ea7a87351f7da4298, REOPEN/MOVE because future has completed 2024-11-23T15:41:41,303 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T15:41:41,303 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194}] 2024-11-23T15:41:41,455 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(122): Close 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:41,455 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T15:41:41,455 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1722): Closing 6680d3b6fc70270ea7a87351f7da4298, disabling compactions & flushes 2024-11-23T15:41:41,456 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:41,456 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:41,456 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. after waiting 0 ms 2024-11-23T15:41:41,456 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:41,461 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-23T15:41:41,462 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:41,462 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegion(1676): Region close journal for 6680d3b6fc70270ea7a87351f7da4298: Waiting for close lock at 1732376501455Running coprocessor pre-close hooks at 1732376501455Disabling compacts and flushes for region at 1732376501455Disabling writes for close at 1732376501456 (+1 ms)Writing region close event to WAL at 1732376501456Running coprocessor post-close hooks at 1732376501462 (+6 ms)Closed at 1732376501462 2024-11-23T15:41:41,462 WARN [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] regionserver.HRegionServer(3299): Not adding moved region record: 6680d3b6fc70270ea7a87351f7da4298 to self. 2024-11-23T15:41:41,465 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=101}] handler.UnassignRegionHandler(157): Closed 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:41,466 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=6680d3b6fc70270ea7a87351f7da4298, regionState=CLOSED 2024-11-23T15:41:41,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=101, ppid=100, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 because future has completed 2024-11-23T15:41:41,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-23T15:41:41,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; CloseRegionProcedure 6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 in 167 msec 2024-11-23T15:41:41,473 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6680d3b6fc70270ea7a87351f7da4298, REOPEN/MOVE; state=CLOSED, location=b712f9af2c12,46589,1732376409194; forceNewPlan=false, retain=true 2024-11-23T15:41:41,624 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=6680d3b6fc70270ea7a87351f7da4298, regionState=OPENING, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:41,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=99, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6680d3b6fc70270ea7a87351f7da4298, REOPEN/MOVE because future has completed 2024-11-23T15:41:41,627 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194}] 2024-11-23T15:41:41,788 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:41,788 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7752): Opening region: {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:41:41,789 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:41,789 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:41:41,789 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7794): checking encryption for 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:41,789 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(7797): checking classloading for 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:41,791 INFO [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:41,792 INFO [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6680d3b6fc70270ea7a87351f7da4298 columnFamilyName A 2024-11-23T15:41:41,793 DEBUG [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:41,794 INFO [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] regionserver.HStore(327): Store=6680d3b6fc70270ea7a87351f7da4298/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:41:41,794 INFO [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:41,795 INFO [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6680d3b6fc70270ea7a87351f7da4298 columnFamilyName B 2024-11-23T15:41:41,795 DEBUG [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:41,796 INFO [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] regionserver.HStore(327): Store=6680d3b6fc70270ea7a87351f7da4298/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:41:41,796 INFO [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:41,797 INFO [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6680d3b6fc70270ea7a87351f7da4298 columnFamilyName C 2024-11-23T15:41:41,797 DEBUG [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:41,797 INFO [StoreOpener-6680d3b6fc70270ea7a87351f7da4298-1 {}] regionserver.HStore(327): Store=6680d3b6fc70270ea7a87351f7da4298/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:41:41,797 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1038): replaying wal for 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:41,798 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:41,800 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:41,801 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1048): stopping wal replay for 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:41,801 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1060): Cleaning up temporary data for 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:41,802 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T15:41:41,804 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1093): writing seq id for 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:41,805 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1114): Opened 6680d3b6fc70270ea7a87351f7da4298; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59228092, jitterRate=-0.11743265390396118}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T15:41:41,805 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:41,806 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegion(1006): Region open journal for 6680d3b6fc70270ea7a87351f7da4298: Running coprocessor pre-open hook at 1732376501789Writing region info on filesystem at 1732376501789Initializing all the Stores at 1732376501790 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376501790Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376501790Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376501790Cleaning up temporary data from old regions at 1732376501801 (+11 ms)Running coprocessor post-open hooks at 1732376501805 (+4 ms)Region opened successfully at 1732376501806 (+1 ms) 2024-11-23T15:41:41,807 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., pid=102, masterSystemTime=1732376501781 2024-11-23T15:41:41,809 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:41,809 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=102}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:41,810 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=100 updating hbase:meta row=6680d3b6fc70270ea7a87351f7da4298, regionState=OPEN, openSeqNum=5, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:41,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=100, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 because future has completed 2024-11-23T15:41:41,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=100 2024-11-23T15:41:41,815 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=100, state=SUCCESS, hasLock=false; OpenRegionProcedure 6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 in 185 msec 2024-11-23T15:41:41,817 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-23T15:41:41,817 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6680d3b6fc70270ea7a87351f7da4298, REOPEN/MOVE in 515 msec 2024-11-23T15:41:41,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-23T15:41:41,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 521 msec 2024-11-23T15:41:41,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 949 msec 2024-11-23T15:41:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=98 2024-11-23T15:41:41,824 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:41,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,825 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:41:41,825 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:41:41,826 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:41:41,826 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:41:41,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:41:41,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@240cebca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:41:41,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:41:41,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:41,828 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43878, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:41:41,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c7d5fbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,829 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:41:41,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:41:41,830 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:41:41,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:41:41,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:41:41,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@330cc692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:41:41,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:41:41,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:41,832 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:41:41,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f714eeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:41:41,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:41:41,834 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:41:41,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:41:41,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:41:41,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a882a8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:41:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:41:41,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:41,836 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:41:41,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42b656b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,837 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:41:41,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:41:41,838 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:41:41,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:41:41,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:41:41,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65a18d07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:41:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:41:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:41,840 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:41:41,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ee3e2bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f004f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,841 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:41:41,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:41:41,842 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:41:41,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:41:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:41:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e534da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:41:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:41:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:41,844 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:41:41,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1db3cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,845 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:41:41,845 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:41:41,846 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:41:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:41:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:41:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59e0ee0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:41:41,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:41:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:41,847 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:41:41,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c686bc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,848 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:41:41,848 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:41:41,849 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:41:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:41:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:41:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@538a8f4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:41:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:41:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:41,850 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44006, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:41:41,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e60f877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,851 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:41:41,851 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:41:41,852 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:41:41,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:41:41,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:41:41,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2421d638, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:41:41,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:41:41,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:41,853 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:41:41,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45dfba7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,854 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,854 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:41:41,855 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:41:41,855 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:41:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:41:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:41:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@480c00de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:41:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:41:41,857 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:41,857 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:41:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c66c8e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa71035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,859 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:41:41,859 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:41:41,860 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:41:41,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:41:41,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:41:41,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6673f4ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,860 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:41:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:41:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:41:41,862 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44060, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:41:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c50b42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:41:41,863 DEBUG [Thread-1576 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:41:41,863 DEBUG [Thread-1578 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:41:41,864 DEBUG [Thread-1572 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:41:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:41:41,864 DEBUG [Thread-1563 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:41:41,864 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:41:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:41:41,864 DEBUG [Thread-1570 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:41:41,864 DEBUG [Thread-1561 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:41:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:41:41,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:41:41,865 DEBUG [Thread-1559 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:41:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:41:41,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-23T15:41:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:41:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:41:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:41:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:41:41,865 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:41:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:41:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:41:41,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:41:41,866 DEBUG [Thread-1565 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:41:41,866 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:41:41,866 DEBUG [Thread-1574 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:41:41,866 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:41:41,866 DEBUG [Thread-1567 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:41:41,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:41:41,867 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:41:41,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:41:41,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:41:41,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:41:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:41:41,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:41:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:41:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:41:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:41:41,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:41:41,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:41:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T15:41:41,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:41:41,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:41:41,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:41:41,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:41:41,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:41:41,870 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:41:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:41:41,871 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:41:41,871 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:41:41,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:41:41,871 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:41:41,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:41:41,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:41:41,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:41:41,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:41:41,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:41:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:41,908 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:41:41,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6680d3b6fc70270ea7a87351f7da4298 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T15:41:41,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112341f75c3f9d094e0690b76e20f7fd119f_6680d3b6fc70270ea7a87351f7da4298 is 50, key is test_row_0/A:col10/1732376501879/Put/seqid=0 2024-11-23T15:41:41,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742168_1344 (size=14594) 2024-11-23T15:41:41,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376561941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:41,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:41,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:41,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376561948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:41,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:41,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:41,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:41,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376561948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:41,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:41,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:41,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:41,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376561949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:41,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376561949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:41,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T15:41:42,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T15:41:42,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:42,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:42,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:42,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:42,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:42,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376562054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:42,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376562064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:42,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:42,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376562064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:42,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:42,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:42,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376562064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:42,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:42,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:42,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:42,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376562064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:42,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:42,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T15:41:42,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:42,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:42,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:42,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:42,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:42,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T15:41:42,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:42,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376562264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:42,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:42,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376562274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:42,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376562274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:42,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376562274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:42,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:42,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:42,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:42,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376562274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:42,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:42,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T15:41:42,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:42,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:42,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:42,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:42,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:42,337 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:42,340 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112341f75c3f9d094e0690b76e20f7fd119f_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112341f75c3f9d094e0690b76e20f7fd119f_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:42,341 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/9f56b2d9097140a091a2f01adb92df3c, store: [table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:41:42,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/9f56b2d9097140a091a2f01adb92df3c is 175, key is test_row_0/A:col10/1732376501879/Put/seqid=0 2024-11-23T15:41:42,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742169_1345 (size=39549) 2024-11-23T15:41:42,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T15:41:42,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:42,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:42,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:42,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:42,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:42,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:42,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T15:41:42,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:42,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376562574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:42,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:42,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376562584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376562584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:42,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376562584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:42,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376562584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:42,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:42,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T15:41:42,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:42,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:42,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:42,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:42,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:42,753 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/9f56b2d9097140a091a2f01adb92df3c 2024-11-23T15:41:42,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/0e457f9baa504bcd8b055fdfa40a4977 is 50, key is test_row_0/B:col10/1732376501879/Put/seqid=0 2024-11-23T15:41:42,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742170_1346 (size=12001) 2024-11-23T15:41:42,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T15:41:42,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:42,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:42,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:42,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:42,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:42,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T15:41:42,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:42,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:42,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:42,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:42,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:43,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T15:41:43,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376563084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:43,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:43,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:43,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:43,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T15:41:43,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:43,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:43,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:43,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:43,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:43,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:43,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376563094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:43,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376563094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:43,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376563094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:43,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:43,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:43,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376563094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:43,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:43,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:43,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:43,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/0e457f9baa504bcd8b055fdfa40a4977 2024-11-23T15:41:43,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/47bf541253e14ca7abb60e39b4ededed is 50, key is test_row_0/C:col10/1732376501879/Put/seqid=0 2024-11-23T15:41:43,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742171_1347 (size=12001) 2024-11-23T15:41:43,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T15:41:43,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:43,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:43,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:43,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:43,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:43,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:43,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T15:41:43,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:43,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:43,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:43,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:43,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:43,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:43,473 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T15:41:43,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T15:41:43,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:43,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:43,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:43,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:43,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:43,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/47bf541253e14ca7abb60e39b4ededed 2024-11-23T15:41:43,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/9f56b2d9097140a091a2f01adb92df3c as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/9f56b2d9097140a091a2f01adb92df3c 2024-11-23T15:41:43,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/9f56b2d9097140a091a2f01adb92df3c, entries=200, sequenceid=17, filesize=38.6 K 2024-11-23T15:41:43,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/0e457f9baa504bcd8b055fdfa40a4977 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/0e457f9baa504bcd8b055fdfa40a4977 2024-11-23T15:41:43,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/0e457f9baa504bcd8b055fdfa40a4977, entries=150, sequenceid=17, filesize=11.7 K 2024-11-23T15:41:43,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/47bf541253e14ca7abb60e39b4ededed as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/47bf541253e14ca7abb60e39b4ededed 2024-11-23T15:41:43,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/47bf541253e14ca7abb60e39b4ededed, entries=150, sequenceid=17, filesize=11.7 K 2024-11-23T15:41:43,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 6680d3b6fc70270ea7a87351f7da4298 in 1716ms, sequenceid=17, compaction requested=false 2024-11-23T15:41:43,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6680d3b6fc70270ea7a87351f7da4298: 2024-11-23T15:41:43,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T15:41:43,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:43,703 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 6680d3b6fc70270ea7a87351f7da4298 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T15:41:43,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e1d8b3041f1d43eca1de9d4739021ada_6680d3b6fc70270ea7a87351f7da4298 is 50, key is test_row_0/A:col10/1732376501948/Put/seqid=0 2024-11-23T15:41:43,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742172_1348 (size=12154) 2024-11-23T15:41:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T15:41:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:44,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:44,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:44,117 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e1d8b3041f1d43eca1de9d4739021ada_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e1d8b3041f1d43eca1de9d4739021ada_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:44,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/5d18c4e0d8a94a56b6ab0e5e87780ef8, store: [table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:41:44,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/5d18c4e0d8a94a56b6ab0e5e87780ef8 is 175, key is test_row_0/A:col10/1732376501948/Put/seqid=0 2024-11-23T15:41:44,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742173_1349 (size=30955) 2024-11-23T15:41:44,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376564118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:44,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:44,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:44,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376564118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:44,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:44,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:44,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376564123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:44,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:44,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376564124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:44,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:44,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376564126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:44,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:44,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:44,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:44,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376564234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:44,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:44,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376564234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:44,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376564234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376564234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:44,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376564235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:44,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:44,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376564444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:44,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376564444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:44,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376564444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:44,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:44,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376564444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:44,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:44,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376564444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:44,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:44,522 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/5d18c4e0d8a94a56b6ab0e5e87780ef8 2024-11-23T15:41:44,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/a38db9c3c70042bc99df257bc503a374 is 50, key is test_row_0/B:col10/1732376501948/Put/seqid=0 2024-11-23T15:41:44,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742174_1350 (size=12001) 2024-11-23T15:41:44,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:44,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376564754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:44,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376564754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:44,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376564754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:44,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:44,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376564754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:44,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:44,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:44,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376564754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:44,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:44,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:44,932 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/a38db9c3c70042bc99df257bc503a374 2024-11-23T15:41:44,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/863fd9a825cb41c081f6a82e625696bb is 50, key is test_row_0/C:col10/1732376501948/Put/seqid=0 2024-11-23T15:41:44,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742175_1351 (size=12001) 2024-11-23T15:41:45,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:45,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376565264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:45,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:45,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:45,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376565264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:45,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:45,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:45,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:45,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:45,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:45,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376565264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:45,268 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:45,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:45,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:45,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:45,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376565274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:45,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376565274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:45,341 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/863fd9a825cb41c081f6a82e625696bb 2024-11-23T15:41:45,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/5d18c4e0d8a94a56b6ab0e5e87780ef8 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/5d18c4e0d8a94a56b6ab0e5e87780ef8 2024-11-23T15:41:45,353 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/5d18c4e0d8a94a56b6ab0e5e87780ef8, entries=150, sequenceid=41, filesize=30.2 K 2024-11-23T15:41:45,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/a38db9c3c70042bc99df257bc503a374 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/a38db9c3c70042bc99df257bc503a374 2024-11-23T15:41:45,358 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/a38db9c3c70042bc99df257bc503a374, entries=150, sequenceid=41, filesize=11.7 K 2024-11-23T15:41:45,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/863fd9a825cb41c081f6a82e625696bb as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/863fd9a825cb41c081f6a82e625696bb 2024-11-23T15:41:45,362 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/863fd9a825cb41c081f6a82e625696bb, entries=150, sequenceid=41, filesize=11.7 K 2024-11-23T15:41:45,363 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 6680d3b6fc70270ea7a87351f7da4298 in 1660ms, sequenceid=41, compaction requested=false 2024-11-23T15:41:45,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 6680d3b6fc70270ea7a87351f7da4298: 2024-11-23T15:41:45,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:45,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-23T15:41:45,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-23T15:41:45,366 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-23T15:41:45,366 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4970 sec 2024-11-23T15:41:45,367 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 3.5030 sec 2024-11-23T15:41:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T15:41:46,044 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:46,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:41:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-23T15:41:46,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T15:41:46,046 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:41:46,047 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:41:46,047 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:41:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T15:41:46,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-23T15:41:46,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:46,199 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 6680d3b6fc70270ea7a87351f7da4298 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T15:41:46,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123200e331e12b048bd933a5e2bda411014_6680d3b6fc70270ea7a87351f7da4298 is 50, key is test_row_0/A:col10/1732376504117/Put/seqid=0 2024-11-23T15:41:46,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742176_1352 (size=12154) 2024-11-23T15:41:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:46,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:46,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376566316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:46,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:46,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376566318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:46,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376566319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376566325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:46,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376566327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T15:41:46,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376566434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:46,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376566434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:46,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376566434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:46,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376566434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:46,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:46,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376566444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:46,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:46,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:46,612 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123200e331e12b048bd933a5e2bda411014_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123200e331e12b048bd933a5e2bda411014_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:46,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/2e876db9b873453ba65376d35201a6df, store: [table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:41:46,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/2e876db9b873453ba65376d35201a6df is 175, key is test_row_0/A:col10/1732376504117/Put/seqid=0 2024-11-23T15:41:46,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742177_1353 (size=30955) 2024-11-23T15:41:46,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:46,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376566644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:46,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:46,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:46,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376566644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:46,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:46,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:46,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376566644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:46,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:46,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376566644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:46,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:46,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:46,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376566664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:46,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T15:41:46,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:46,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376566954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:46,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:46,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376566954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:46,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:46,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376566954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:46,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:46,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376566954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:46,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:46,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376566974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:46,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:46,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:47,017 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/2e876db9b873453ba65376d35201a6df 2024-11-23T15:41:47,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/54c9678b6de946f8a2fe34ff95b05825 is 50, key is test_row_0/B:col10/1732376504117/Put/seqid=0 2024-11-23T15:41:47,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742178_1354 (size=12001) 2024-11-23T15:41:47,028 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/54c9678b6de946f8a2fe34ff95b05825 2024-11-23T15:41:47,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/7218910670354257aea83ae9fbfeb5fb is 50, key is test_row_0/C:col10/1732376504117/Put/seqid=0 2024-11-23T15:41:47,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742179_1355 (size=12001) 2024-11-23T15:41:47,037 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/7218910670354257aea83ae9fbfeb5fb 2024-11-23T15:41:47,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/2e876db9b873453ba65376d35201a6df as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/2e876db9b873453ba65376d35201a6df 2024-11-23T15:41:47,045 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/2e876db9b873453ba65376d35201a6df, entries=150, sequenceid=53, filesize=30.2 K 2024-11-23T15:41:47,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/54c9678b6de946f8a2fe34ff95b05825 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/54c9678b6de946f8a2fe34ff95b05825 2024-11-23T15:41:47,049 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/54c9678b6de946f8a2fe34ff95b05825, entries=150, sequenceid=53, filesize=11.7 K 2024-11-23T15:41:47,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/7218910670354257aea83ae9fbfeb5fb as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/7218910670354257aea83ae9fbfeb5fb 2024-11-23T15:41:47,054 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/7218910670354257aea83ae9fbfeb5fb, entries=150, sequenceid=53, filesize=11.7 K 2024-11-23T15:41:47,055 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6680d3b6fc70270ea7a87351f7da4298 in 856ms, sequenceid=53, compaction requested=true 2024-11-23T15:41:47,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 6680d3b6fc70270ea7a87351f7da4298: 2024-11-23T15:41:47,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:47,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-23T15:41:47,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-23T15:41:47,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-23T15:41:47,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0100 sec 2024-11-23T15:41:47,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.0140 sec 2024-11-23T15:41:47,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T15:41:47,184 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:47,185 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:41:47,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-23T15:41:47,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-23T15:41:47,186 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:41:47,187 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:41:47,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:41:47,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-23T15:41:47,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-23T15:41:47,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:47,339 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 6680d3b6fc70270ea7a87351f7da4298 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T15:41:47,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411236bfe8e65c8dc45ac9df9ea9a95e530f5_6680d3b6fc70270ea7a87351f7da4298 is 50, key is test_row_0/A:col10/1732376506317/Put/seqid=0 2024-11-23T15:41:47,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742180_1356 (size=12154) 2024-11-23T15:41:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:47,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:47,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376567480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:47,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376567481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:47,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376567481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376567481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:47,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376567494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-23T15:41:47,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376567594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:47,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376567594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376567594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:47,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376567594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:47,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:47,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:47,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:47,760 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411236bfe8e65c8dc45ac9df9ea9a95e530f5_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236bfe8e65c8dc45ac9df9ea9a95e530f5_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:47,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/2ced121e989f4deab318159e8dad2cdf, store: [table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:41:47,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/2ced121e989f4deab318159e8dad2cdf is 175, key is test_row_0/A:col10/1732376506317/Put/seqid=0 2024-11-23T15:41:47,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742181_1357 (size=30955) 2024-11-23T15:41:47,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376567804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:47,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:47,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376567804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:47,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:47,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376567804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:47,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-23T15:41:47,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376567814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:48,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:48,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376568114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:48,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:48,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:48,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376568124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:48,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:48,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:48,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:48,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:48,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376568124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:48,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:48,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:48,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:48,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:48,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376568124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:48,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:48,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:48,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:48,165 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/2ced121e989f4deab318159e8dad2cdf 2024-11-23T15:41:48,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/e4c5a7b081ee4ba5992301c34e83436f is 50, key is test_row_0/B:col10/1732376506317/Put/seqid=0 2024-11-23T15:41:48,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742182_1358 (size=12001) 2024-11-23T15:41:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-23T15:41:48,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376568514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:48,519 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_1' in 6680d3b6fc70270ea7a87351f7da4298 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:41:48,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:48,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:48,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:48,574 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/e4c5a7b081ee4ba5992301c34e83436f 2024-11-23T15:41:48,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/655855cfa4cd4243a871ef4d0e99588d is 50, key is test_row_0/C:col10/1732376506317/Put/seqid=0 2024-11-23T15:41:48,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742183_1359 (size=12001) 2024-11-23T15:41:48,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:48,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376568634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:48,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:48,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:48,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376568634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:48,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:48,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:48,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:48,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:48,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:48,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376568644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:48,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376568645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:48,768 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-23T15:41:48,768 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-23T15:41:48,984 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/655855cfa4cd4243a871ef4d0e99588d 2024-11-23T15:41:48,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/2ced121e989f4deab318159e8dad2cdf as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/2ced121e989f4deab318159e8dad2cdf 2024-11-23T15:41:48,991 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/2ced121e989f4deab318159e8dad2cdf, entries=150, sequenceid=77, filesize=30.2 K 2024-11-23T15:41:48,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/e4c5a7b081ee4ba5992301c34e83436f as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/e4c5a7b081ee4ba5992301c34e83436f 2024-11-23T15:41:48,994 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/e4c5a7b081ee4ba5992301c34e83436f, entries=150, sequenceid=77, filesize=11.7 K 2024-11-23T15:41:48,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/655855cfa4cd4243a871ef4d0e99588d as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/655855cfa4cd4243a871ef4d0e99588d 2024-11-23T15:41:48,998 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/655855cfa4cd4243a871ef4d0e99588d, entries=150, sequenceid=77, filesize=11.7 K 2024-11-23T15:41:49,000 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 6680d3b6fc70270ea7a87351f7da4298 in 1661ms, sequenceid=77, compaction requested=true 2024-11-23T15:41:49,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 6680d3b6fc70270ea7a87351f7da4298: 2024-11-23T15:41:49,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:49,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-23T15:41:49,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-23T15:41:49,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-23T15:41:49,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8140 sec 2024-11-23T15:41:49,004 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.8180 sec 2024-11-23T15:41:49,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-23T15:41:49,334 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:49,334 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:41:49,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-23T15:41:49,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-23T15:41:49,336 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:41:49,336 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:41:49,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:41:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-23T15:41:49,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-23T15:41:49,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:49,489 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 6680d3b6fc70270ea7a87351f7da4298 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T15:41:49,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123527976d74f314486babd4d3350a1d661_6680d3b6fc70270ea7a87351f7da4298 is 50, key is test_row_0/A:col10/1732376507469/Put/seqid=0 2024-11-23T15:41:49,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742184_1360 (size=12154) 2024-11-23T15:41:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-23T15:41:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:49,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:49,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376569711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:49,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:49,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:49,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:49,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376569712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:49,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376569713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:49,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:49,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:49,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:49,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376569717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:49,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:49,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:49,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:49,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:49,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376569824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:49,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:49,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:49,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:49,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376569824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:49,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:49,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:49,833 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:49,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376569834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:49,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:49,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:49,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:49,844 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376569834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:49,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:49,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:49,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:49,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:49,907 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123527976d74f314486babd4d3350a1d661_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123527976d74f314486babd4d3350a1d661_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:49,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/d5db65e606da4ab083fe11d648ea277c, store: [table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:41:49,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/d5db65e606da4ab083fe11d648ea277c is 175, key is test_row_0/A:col10/1732376507469/Put/seqid=0 2024-11-23T15:41:49,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742185_1361 (size=30955) 2024-11-23T15:41:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-23T15:41:50,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376570044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:50,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376570044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376570054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:50,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376570054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:50,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:50,313 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=89, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/d5db65e606da4ab083fe11d648ea277c 2024-11-23T15:41:50,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/017d466bd9f64fc38b2e2542c1556392 is 50, key is test_row_0/B:col10/1732376507469/Put/seqid=0 2024-11-23T15:41:50,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742186_1362 (size=12001) 2024-11-23T15:41:50,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:50,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376570364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:50,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:50,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376570364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:50,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:50,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:50,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:50,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376570364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:50,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376570364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-23T15:41:50,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:50,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376570534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:50,542 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_1' in 6680d3b6fc70270ea7a87351f7da4298 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:41:50,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:50,725 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/017d466bd9f64fc38b2e2542c1556392 2024-11-23T15:41:50,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/4add009d8eb24a8e819dde3c36922c12 is 50, key is test_row_0/C:col10/1732376507469/Put/seqid=0 2024-11-23T15:41:50,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742187_1363 (size=12001) 2024-11-23T15:41:50,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376570874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:50,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:50,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376570874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:50,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:50,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:50,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376570875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376570875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:51,134 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/4add009d8eb24a8e819dde3c36922c12 2024-11-23T15:41:51,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/d5db65e606da4ab083fe11d648ea277c as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/d5db65e606da4ab083fe11d648ea277c 2024-11-23T15:41:51,140 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/d5db65e606da4ab083fe11d648ea277c, entries=150, sequenceid=89, filesize=30.2 K 2024-11-23T15:41:51,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/017d466bd9f64fc38b2e2542c1556392 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/017d466bd9f64fc38b2e2542c1556392 2024-11-23T15:41:51,144 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/017d466bd9f64fc38b2e2542c1556392, entries=150, sequenceid=89, filesize=11.7 K 2024-11-23T15:41:51,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/4add009d8eb24a8e819dde3c36922c12 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/4add009d8eb24a8e819dde3c36922c12 2024-11-23T15:41:51,148 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/4add009d8eb24a8e819dde3c36922c12, entries=150, sequenceid=89, filesize=11.7 K 2024-11-23T15:41:51,149 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6680d3b6fc70270ea7a87351f7da4298 in 1660ms, sequenceid=89, compaction requested=true 2024-11-23T15:41:51,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 6680d3b6fc70270ea7a87351f7da4298: 2024-11-23T15:41:51,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:51,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-23T15:41:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-23T15:41:51,158 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-23T15:41:51,158 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8190 sec 2024-11-23T15:41:51,160 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.8250 sec 2024-11-23T15:41:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-23T15:41:51,495 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:51,495 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:41:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-23T15:41:51,496 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:41:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-23T15:41:51,497 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:41:51,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:41:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-23T15:41:51,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-23T15:41:51,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:51,650 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 6680d3b6fc70270ea7a87351f7da4298 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T15:41:51,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123cd9163fb33dd442093b0dafe13f457e1_6680d3b6fc70270ea7a87351f7da4298 is 50, key is test_row_0/A:col10/1732376509681/Put/seqid=0 2024-11-23T15:41:51,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742188_1364 (size=12154) 2024-11-23T15:41:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-23T15:41:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:51,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:51,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376571906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:51,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:51,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:51,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:51,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376571910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:51,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:51,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:51,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:51,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376571919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:51,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:51,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376571919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:51,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:51,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:51,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:51,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:51,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:52,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:52,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376572024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:52,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:52,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:52,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376572024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:52,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:52,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:52,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376572034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:52,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:52,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:52,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376572034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:52,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:52,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,063 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123cd9163fb33dd442093b0dafe13f457e1_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123cd9163fb33dd442093b0dafe13f457e1_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:52,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/3440e7bd369d47bc8cec4004e6a18a81, store: [table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:41:52,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/3440e7bd369d47bc8cec4004e6a18a81 is 175, key is test_row_0/A:col10/1732376509681/Put/seqid=0 2024-11-23T15:41:52,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742189_1365 (size=30955) 2024-11-23T15:41:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-23T15:41:52,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376572234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:52,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:52,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376572234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:52,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:52,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376572254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376572254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,264 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:52,467 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=113, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/3440e7bd369d47bc8cec4004e6a18a81 2024-11-23T15:41:52,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/d7e41a3e24834da8b45449462c3b16a6 is 50, key is test_row_0/B:col10/1732376509681/Put/seqid=0 2024-11-23T15:41:52,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742190_1366 (size=12001) 2024-11-23T15:41:52,478 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/d7e41a3e24834da8b45449462c3b16a6 2024-11-23T15:41:52,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/5b5dff6e86a14c67bfc94435e363d45b is 50, key is test_row_0/C:col10/1732376509681/Put/seqid=0 2024-11-23T15:41:52,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742191_1367 (size=12001) 2024-11-23T15:41:52,491 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/5b5dff6e86a14c67bfc94435e363d45b 2024-11-23T15:41:52,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/3440e7bd369d47bc8cec4004e6a18a81 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/3440e7bd369d47bc8cec4004e6a18a81 2024-11-23T15:41:52,500 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/3440e7bd369d47bc8cec4004e6a18a81, entries=150, sequenceid=113, filesize=30.2 K 2024-11-23T15:41:52,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/d7e41a3e24834da8b45449462c3b16a6 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/d7e41a3e24834da8b45449462c3b16a6 2024-11-23T15:41:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,504 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/d7e41a3e24834da8b45449462c3b16a6, entries=150, sequenceid=113, filesize=11.7 K 2024-11-23T15:41:52,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/5b5dff6e86a14c67bfc94435e363d45b as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/5b5dff6e86a14c67bfc94435e363d45b 2024-11-23T15:41:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,509 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/5b5dff6e86a14c67bfc94435e363d45b, entries=150, sequenceid=113, filesize=11.7 K 2024-11-23T15:41:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,510 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 6680d3b6fc70270ea7a87351f7da4298 in 860ms, sequenceid=113, compaction requested=true 2024-11-23T15:41:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 6680d3b6fc70270ea7a87351f7da4298: 2024-11-23T15:41:52,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:52,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-23T15:41:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-23T15:41:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,513 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-23T15:41:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,513 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0140 sec 2024-11-23T15:41:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,514 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.0180 sec 2024-11-23T15:41:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:52,572 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:41:52,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6680d3b6fc70270ea7a87351f7da4298 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-23T15:41:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112317cbfab30e344c3f8c59b5a58c55af72_6680d3b6fc70270ea7a87351f7da4298 is 50, key is test_row_0/A:col10/1732376511918/Put/seqid=0 2024-11-23T15:41:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742192_1368 (size=12154) 2024-11-23T15:41:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-23T15:41:52,634 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:52,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:41:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-23T15:41:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T15:41:52,636 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:41:52,637 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:41:52,637 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:41:52,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376572665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:52,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:52,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:52,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376572666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:52,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:52,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376572669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:52,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:52,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376572670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:52,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T15:41:52,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-23T15:41:52,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:52,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:52,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:52,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:52,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:52,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376572784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:52,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:52,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376572784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:52,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376572794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:52,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376572794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:52,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:52,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-23T15:41:52,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:52,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:52,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:52,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:52,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:52,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T15:41:52,988 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:52,991 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112317cbfab30e344c3f8c59b5a58c55af72_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112317cbfab30e344c3f8c59b5a58c55af72_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:52,992 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/99702eeab3f6416dad1602d8b105f580, store: [table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:41:52,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/99702eeab3f6416dad1602d8b105f580 is 175, key is test_row_0/A:col10/1732376511918/Put/seqid=0 2024-11-23T15:41:52,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742193_1369 (size=30951) 2024-11-23T15:41:53,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376572994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376572994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:53,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:53,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376573014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:53,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376573014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:53,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-23T15:41:53,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:53,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:53,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:53,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:53,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:53,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:53,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-23T15:41:53,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:53,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:53,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:53,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:53,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:53,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:53,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T15:41:53,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376573315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:53,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:53,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376573317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:53,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:53,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376573334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:53,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376573334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:53,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:53,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:53,397 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=126, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/99702eeab3f6416dad1602d8b105f580 2024-11-23T15:41:53,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-23T15:41:53,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:53,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:53,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:53,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:53,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:53,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:53,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/52b8533475a244f6beabf7322ba161bc is 50, key is test_row_0/B:col10/1732376511918/Put/seqid=0 2024-11-23T15:41:53,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742194_1370 (size=9657) 2024-11-23T15:41:53,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-23T15:41:53,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:53,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:53,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:53,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:53,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:53,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:53,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-23T15:41:53,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:53,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:53,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:53,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:53,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:53,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T15:41:53,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/52b8533475a244f6beabf7322ba161bc 2024-11-23T15:41:53,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/a687b7736cc8484ea0143d15d9d0951a is 50, key is test_row_0/C:col10/1732376511918/Put/seqid=0 2024-11-23T15:41:53,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742195_1371 (size=9657) 2024-11-23T15:41:53,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376573834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:53,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376573834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:53,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-23T15:41:53,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:53,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:53,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:53,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:53,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:53,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376573854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:53,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376573854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:53,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:53,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:54,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-23T15:41:54,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:54,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:54,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:54,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:54,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:54,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:54,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-23T15:41:54,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:54,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:54,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:54,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:54,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:54,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:54,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/a687b7736cc8484ea0143d15d9d0951a 2024-11-23T15:41:54,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/99702eeab3f6416dad1602d8b105f580 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/99702eeab3f6416dad1602d8b105f580 2024-11-23T15:41:54,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/99702eeab3f6416dad1602d8b105f580, entries=150, sequenceid=126, filesize=30.2 K 2024-11-23T15:41:54,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/52b8533475a244f6beabf7322ba161bc as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/52b8533475a244f6beabf7322ba161bc 2024-11-23T15:41:54,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/52b8533475a244f6beabf7322ba161bc, entries=100, sequenceid=126, filesize=9.4 K 2024-11-23T15:41:54,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/a687b7736cc8484ea0143d15d9d0951a as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/a687b7736cc8484ea0143d15d9d0951a 2024-11-23T15:41:54,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/a687b7736cc8484ea0143d15d9d0951a, entries=100, sequenceid=126, filesize=9.4 K 2024-11-23T15:41:54,245 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 6680d3b6fc70270ea7a87351f7da4298 in 1673ms, sequenceid=126, compaction requested=true 2024-11-23T15:41:54,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6680d3b6fc70270ea7a87351f7da4298: 2024-11-23T15:41:54,246 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-23T15:41:54,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6680d3b6fc70270ea7a87351f7da4298:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:41:54,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:54,246 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-23T15:41:54,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6680d3b6fc70270ea7a87351f7da4298:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:41:54,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:54,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6680d3b6fc70270ea7a87351f7da4298:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:41:54,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:54,247 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 225275 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-23T15:41:54,247 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 81663 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-23T15:41:54,247 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): 6680d3b6fc70270ea7a87351f7da4298/B is initiating minor compaction (all files) 2024-11-23T15:41:54,247 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): 6680d3b6fc70270ea7a87351f7da4298/A is initiating minor compaction (all files) 2024-11-23T15:41:54,247 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6680d3b6fc70270ea7a87351f7da4298/B in TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:54,247 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6680d3b6fc70270ea7a87351f7da4298/A in TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:54,247 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/0e457f9baa504bcd8b055fdfa40a4977, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/a38db9c3c70042bc99df257bc503a374, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/54c9678b6de946f8a2fe34ff95b05825, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/e4c5a7b081ee4ba5992301c34e83436f, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/017d466bd9f64fc38b2e2542c1556392, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/d7e41a3e24834da8b45449462c3b16a6, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/52b8533475a244f6beabf7322ba161bc] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp, totalSize=79.7 K 2024-11-23T15:41:54,247 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/9f56b2d9097140a091a2f01adb92df3c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/5d18c4e0d8a94a56b6ab0e5e87780ef8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/2e876db9b873453ba65376d35201a6df, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/2ced121e989f4deab318159e8dad2cdf, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/d5db65e606da4ab083fe11d648ea277c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/3440e7bd369d47bc8cec4004e6a18a81, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/99702eeab3f6416dad1602d8b105f580] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp, totalSize=220.0 K 2024-11-23T15:41:54,247 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=9 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:54,247 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. files: [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/9f56b2d9097140a091a2f01adb92df3c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/5d18c4e0d8a94a56b6ab0e5e87780ef8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/2e876db9b873453ba65376d35201a6df, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/2ced121e989f4deab318159e8dad2cdf, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/d5db65e606da4ab083fe11d648ea277c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/3440e7bd369d47bc8cec4004e6a18a81, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/99702eeab3f6416dad1602d8b105f580] 2024-11-23T15:41:54,248 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e457f9baa504bcd8b055fdfa40a4977, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732376501879 2024-11-23T15:41:54,248 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f56b2d9097140a091a2f01adb92df3c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732376501874 2024-11-23T15:41:54,248 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting a38db9c3c70042bc99df257bc503a374, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732376501947 2024-11-23T15:41:54,248 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d18c4e0d8a94a56b6ab0e5e87780ef8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732376501947 2024-11-23T15:41:54,248 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 54c9678b6de946f8a2fe34ff95b05825, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732376504116 2024-11-23T15:41:54,248 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e876db9b873453ba65376d35201a6df, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732376504116 2024-11-23T15:41:54,248 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting e4c5a7b081ee4ba5992301c34e83436f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732376506316 2024-11-23T15:41:54,248 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 017d466bd9f64fc38b2e2542c1556392, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732376507469 2024-11-23T15:41:54,248 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ced121e989f4deab318159e8dad2cdf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732376506316 2024-11-23T15:41:54,249 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting d7e41a3e24834da8b45449462c3b16a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732376509681 2024-11-23T15:41:54,249 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5db65e606da4ab083fe11d648ea277c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732376507469 2024-11-23T15:41:54,249 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 52b8533475a244f6beabf7322ba161bc, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732376511918 2024-11-23T15:41:54,249 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3440e7bd369d47bc8cec4004e6a18a81, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732376509681 2024-11-23T15:41:54,249 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99702eeab3f6416dad1602d8b105f580, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732376511905 2024-11-23T15:41:54,256 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:41:54,258 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6680d3b6fc70270ea7a87351f7da4298#B#compaction#319 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:54,258 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/d0502014413745d1bb8d5f93267d159e is 50, key is test_row_0/B:col10/1732376511918/Put/seqid=0 2024-11-23T15:41:54,260 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112317c8f4fc60e94b3e871c197667de0074_6680d3b6fc70270ea7a87351f7da4298 store=[table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:41:54,262 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112317c8f4fc60e94b3e871c197667de0074_6680d3b6fc70270ea7a87351f7da4298, store=[table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:41:54,262 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112317c8f4fc60e94b3e871c197667de0074_6680d3b6fc70270ea7a87351f7da4298 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:41:54,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742196_1372 (size=12241) 2024-11-23T15:41:54,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742197_1373 (size=4469) 2024-11-23T15:41:54,273 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6680d3b6fc70270ea7a87351f7da4298#A#compaction#318 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:54,273 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/1905e303e55149feaa67d7666c2636e2 is 175, key is test_row_0/A:col10/1732376511918/Put/seqid=0 2024-11-23T15:41:54,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742198_1374 (size=31302) 2024-11-23T15:41:54,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-23T15:41:54,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:54,317 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 6680d3b6fc70270ea7a87351f7da4298 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T15:41:54,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123899281f6b50c456f95833ea2d6206bbd_6680d3b6fc70270ea7a87351f7da4298 is 50, key is test_row_0/A:col10/1732376512668/Put/seqid=0 2024-11-23T15:41:54,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742199_1375 (size=12304) 2024-11-23T15:41:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:54,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:54,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376574669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:54,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:54,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:54,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:54,676 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/d0502014413745d1bb8d5f93267d159e as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/d0502014413745d1bb8d5f93267d159e 2024-11-23T15:41:54,679 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 6680d3b6fc70270ea7a87351f7da4298/B of 6680d3b6fc70270ea7a87351f7da4298 into d0502014413745d1bb8d5f93267d159e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:54,679 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6680d3b6fc70270ea7a87351f7da4298: 2024-11-23T15:41:54,679 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., storeName=6680d3b6fc70270ea7a87351f7da4298/B, priority=9, startTime=1732376514246; duration=0sec 2024-11-23T15:41:54,679 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:54,679 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6680d3b6fc70270ea7a87351f7da4298:B 2024-11-23T15:41:54,679 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-23T15:41:54,680 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 81663 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-23T15:41:54,681 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): 6680d3b6fc70270ea7a87351f7da4298/C is initiating minor compaction (all files) 2024-11-23T15:41:54,681 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6680d3b6fc70270ea7a87351f7da4298/C in TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:54,681 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/47bf541253e14ca7abb60e39b4ededed, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/863fd9a825cb41c081f6a82e625696bb, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/7218910670354257aea83ae9fbfeb5fb, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/655855cfa4cd4243a871ef4d0e99588d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/4add009d8eb24a8e819dde3c36922c12, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/5b5dff6e86a14c67bfc94435e363d45b, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/a687b7736cc8484ea0143d15d9d0951a] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp, totalSize=79.7 K 2024-11-23T15:41:54,681 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 47bf541253e14ca7abb60e39b4ededed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732376501879 2024-11-23T15:41:54,681 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 863fd9a825cb41c081f6a82e625696bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732376501947 2024-11-23T15:41:54,681 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 7218910670354257aea83ae9fbfeb5fb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732376504116 2024-11-23T15:41:54,682 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 655855cfa4cd4243a871ef4d0e99588d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732376506316 2024-11-23T15:41:54,682 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 4add009d8eb24a8e819dde3c36922c12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732376507469 2024-11-23T15:41:54,682 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b5dff6e86a14c67bfc94435e363d45b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1732376509681 2024-11-23T15:41:54,682 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting a687b7736cc8484ea0143d15d9d0951a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732376511918 2024-11-23T15:41:54,683 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/1905e303e55149feaa67d7666c2636e2 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/1905e303e55149feaa67d7666c2636e2 2024-11-23T15:41:54,687 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 6680d3b6fc70270ea7a87351f7da4298/A of 6680d3b6fc70270ea7a87351f7da4298 into 1905e303e55149feaa67d7666c2636e2(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:54,687 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6680d3b6fc70270ea7a87351f7da4298: 2024-11-23T15:41:54,687 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., storeName=6680d3b6fc70270ea7a87351f7da4298/A, priority=9, startTime=1732376514245; duration=0sec 2024-11-23T15:41:54,687 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:54,687 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6680d3b6fc70270ea7a87351f7da4298:A 2024-11-23T15:41:54,693 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6680d3b6fc70270ea7a87351f7da4298#C#compaction#321 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:54,694 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/e1ab255c73704ff6914020ed4db77531 is 50, key is test_row_0/C:col10/1732376511918/Put/seqid=0 2024-11-23T15:41:54,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742200_1376 (size=12241) 2024-11-23T15:41:54,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:54,730 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123899281f6b50c456f95833ea2d6206bbd_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123899281f6b50c456f95833ea2d6206bbd_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:54,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/3c70e8ae27074d9595e95a314581b769, store: [table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:41:54,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/3c70e8ae27074d9595e95a314581b769 is 175, key is test_row_0/A:col10/1732376512668/Put/seqid=0 2024-11-23T15:41:54,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742201_1377 (size=31105) 2024-11-23T15:41:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T15:41:54,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376574784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:54,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:54,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:54,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:54,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376574854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:54,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376574854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:54,862 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_0' in 6680d3b6fc70270ea7a87351f7da4298 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:41:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:54,862 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_0' in 6680d3b6fc70270ea7a87351f7da4298 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:41:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:54,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376574864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:54,872 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_1' in 6680d3b6fc70270ea7a87351f7da4298 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:41:54,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:54,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:54,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:54,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376574874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:54,882 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_1' in 6680d3b6fc70270ea7a87351f7da4298 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:41:54,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:55,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376574994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:55,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:55,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:55,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:55,102 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/e1ab255c73704ff6914020ed4db77531 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/e1ab255c73704ff6914020ed4db77531 2024-11-23T15:41:55,106 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 6680d3b6fc70270ea7a87351f7da4298/C of 6680d3b6fc70270ea7a87351f7da4298 into e1ab255c73704ff6914020ed4db77531(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:55,106 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6680d3b6fc70270ea7a87351f7da4298: 2024-11-23T15:41:55,107 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., storeName=6680d3b6fc70270ea7a87351f7da4298/C, priority=9, startTime=1732376514246; duration=0sec 2024-11-23T15:41:55,107 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:55,107 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6680d3b6fc70270ea7a87351f7da4298:C 2024-11-23T15:41:55,135 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=149, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/3c70e8ae27074d9595e95a314581b769 2024-11-23T15:41:55,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/7caa71e684294ee6ac85580b450de695 is 50, key is test_row_0/B:col10/1732376512668/Put/seqid=0 2024-11-23T15:41:55,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742202_1378 (size=12151) 2024-11-23T15:41:55,145 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/7caa71e684294ee6ac85580b450de695 2024-11-23T15:41:55,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/150939c969044e8f83833fab71c1139f is 50, key is test_row_0/C:col10/1732376512668/Put/seqid=0 2024-11-23T15:41:55,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742203_1379 (size=12151) 2024-11-23T15:41:55,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:55,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376575304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:55,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:55,554 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/150939c969044e8f83833fab71c1139f 2024-11-23T15:41:55,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/3c70e8ae27074d9595e95a314581b769 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/3c70e8ae27074d9595e95a314581b769 2024-11-23T15:41:55,561 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/3c70e8ae27074d9595e95a314581b769, entries=150, sequenceid=149, filesize=30.4 K 2024-11-23T15:41:55,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/7caa71e684294ee6ac85580b450de695 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/7caa71e684294ee6ac85580b450de695 2024-11-23T15:41:55,565 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/7caa71e684294ee6ac85580b450de695, entries=150, sequenceid=149, filesize=11.9 K 2024-11-23T15:41:55,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/150939c969044e8f83833fab71c1139f as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/150939c969044e8f83833fab71c1139f 2024-11-23T15:41:55,568 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/150939c969044e8f83833fab71c1139f, entries=150, sequenceid=149, filesize=11.9 K 2024-11-23T15:41:55,569 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 6680d3b6fc70270ea7a87351f7da4298 in 1251ms, sequenceid=149, compaction requested=false 2024-11-23T15:41:55,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 6680d3b6fc70270ea7a87351f7da4298: 2024-11-23T15:41:55,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:55,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-23T15:41:55,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-23T15:41:55,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-23T15:41:55,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9330 sec 2024-11-23T15:41:55,573 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.9370 sec 2024-11-23T15:41:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:55,817 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:41:55,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6680d3b6fc70270ea7a87351f7da4298 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T15:41:55,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123f082dcfe72204c80a891dc74faf3af4e_6680d3b6fc70270ea7a87351f7da4298 is 50, key is test_row_0/A:col10/1732376515817/Put/seqid=0 2024-11-23T15:41:55,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742204_1380 (size=14794) 2024-11-23T15:41:56,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:56,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376575991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:56,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:56,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:56,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:56,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376576114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:56,228 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:56,232 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123f082dcfe72204c80a891dc74faf3af4e_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123f082dcfe72204c80a891dc74faf3af4e_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:56,232 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/dc9b3c33f6d34b16bd095325c8c63452, store: [table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:41:56,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/dc9b3c33f6d34b16bd095325c8c63452 is 175, key is test_row_0/A:col10/1732376515817/Put/seqid=0 2024-11-23T15:41:56,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742205_1381 (size=39749) 2024-11-23T15:41:56,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376576324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:56,637 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/dc9b3c33f6d34b16bd095325c8c63452 2024-11-23T15:41:56,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376576634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:56,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:56,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:56,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:56,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/73cf4b5df06c481bb5b3cae88d724c3c is 50, key is test_row_0/B:col10/1732376515817/Put/seqid=0 2024-11-23T15:41:56,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742206_1382 (size=12151) 2024-11-23T15:41:56,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T15:41:56,804 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:56,804 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:41:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-23T15:41:56,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-23T15:41:56,806 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:41:56,807 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:41:56,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:41:56,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376576874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:56,878 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_0' in 6680d3b6fc70270ea7a87351f7da4298 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:41:56,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:56,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:56,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:56,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376576884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:56,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:56,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376576884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:56,892 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_0' in 6680d3b6fc70270ea7a87351f7da4298 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:41:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:56,892 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_1' in 6680d3b6fc70270ea7a87351f7da4298 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:41:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:56,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:56,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:56,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376576904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:56,911 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_1' in 6680d3b6fc70270ea7a87351f7da4298 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:41:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:56,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:56,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-23T15:41:56,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-23T15:41:56,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:56,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:56,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:56,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:56,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:56,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:57,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/73cf4b5df06c481bb5b3cae88d724c3c 2024-11-23T15:41:57,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/05cae72f513a4fdca8190df178bc5e65 is 50, key is test_row_0/C:col10/1732376515817/Put/seqid=0 2024-11-23T15:41:57,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742207_1383 (size=12151) 2024-11-23T15:41:57,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-23T15:41:57,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:57,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:57,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:57,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:57,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-23T15:41:57,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376577144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:57,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:57,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:57,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:57,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-23T15:41:57,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:57,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:57,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:57,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:57,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:57,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-23T15:41:57,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:57,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:57,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:57,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:57,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-23T15:41:57,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/05cae72f513a4fdca8190df178bc5e65 2024-11-23T15:41:57,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/dc9b3c33f6d34b16bd095325c8c63452 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/dc9b3c33f6d34b16bd095325c8c63452 2024-11-23T15:41:57,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/dc9b3c33f6d34b16bd095325c8c63452, entries=200, sequenceid=166, filesize=38.8 K 2024-11-23T15:41:57,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/73cf4b5df06c481bb5b3cae88d724c3c as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/73cf4b5df06c481bb5b3cae88d724c3c 2024-11-23T15:41:57,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/73cf4b5df06c481bb5b3cae88d724c3c, entries=150, sequenceid=166, filesize=11.9 K 2024-11-23T15:41:57,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/05cae72f513a4fdca8190df178bc5e65 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/05cae72f513a4fdca8190df178bc5e65 2024-11-23T15:41:57,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/05cae72f513a4fdca8190df178bc5e65, entries=150, sequenceid=166, filesize=11.9 K 2024-11-23T15:41:57,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 6680d3b6fc70270ea7a87351f7da4298 in 1656ms, sequenceid=166, compaction requested=true 2024-11-23T15:41:57,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6680d3b6fc70270ea7a87351f7da4298: 2024-11-23T15:41:57,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6680d3b6fc70270ea7a87351f7da4298:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:41:57,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:57,475 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:57,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6680d3b6fc70270ea7a87351f7da4298:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:41:57,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:57,475 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:57,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6680d3b6fc70270ea7a87351f7da4298:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:41:57,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:57,475 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102156 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:57,475 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:57,475 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): 6680d3b6fc70270ea7a87351f7da4298/B is initiating minor compaction (all files) 2024-11-23T15:41:57,475 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): 6680d3b6fc70270ea7a87351f7da4298/A is initiating minor compaction (all files) 2024-11-23T15:41:57,476 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6680d3b6fc70270ea7a87351f7da4298/A in TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:57,476 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6680d3b6fc70270ea7a87351f7da4298/B in TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:57,476 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/d0502014413745d1bb8d5f93267d159e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/7caa71e684294ee6ac85580b450de695, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/73cf4b5df06c481bb5b3cae88d724c3c] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp, totalSize=35.7 K 2024-11-23T15:41:57,476 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/1905e303e55149feaa67d7666c2636e2, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/3c70e8ae27074d9595e95a314581b769, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/dc9b3c33f6d34b16bd095325c8c63452] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp, totalSize=99.8 K 2024-11-23T15:41:57,476 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:57,476 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. files: [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/1905e303e55149feaa67d7666c2636e2, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/3c70e8ae27074d9595e95a314581b769, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/dc9b3c33f6d34b16bd095325c8c63452] 2024-11-23T15:41:57,476 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting d0502014413745d1bb8d5f93267d159e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732376509715 2024-11-23T15:41:57,476 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1905e303e55149feaa67d7666c2636e2, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732376509715 2024-11-23T15:41:57,476 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 7caa71e684294ee6ac85580b450de695, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732376512655 2024-11-23T15:41:57,476 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c70e8ae27074d9595e95a314581b769, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732376512655 2024-11-23T15:41:57,476 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 73cf4b5df06c481bb5b3cae88d724c3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732376514658 2024-11-23T15:41:57,477 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc9b3c33f6d34b16bd095325c8c63452, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732376514627 2024-11-23T15:41:57,482 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:41:57,483 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6680d3b6fc70270ea7a87351f7da4298#B#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:57,483 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/08723c521b354836852b0be87b93a24f is 50, key is test_row_0/B:col10/1732376515817/Put/seqid=0 2024-11-23T15:41:57,484 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112321d8e2e83e9c433783063b7df59b5a61_6680d3b6fc70270ea7a87351f7da4298 store=[table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:41:57,486 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112321d8e2e83e9c433783063b7df59b5a61_6680d3b6fc70270ea7a87351f7da4298, store=[table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:41:57,486 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112321d8e2e83e9c433783063b7df59b5a61_6680d3b6fc70270ea7a87351f7da4298 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:41:57,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742208_1384 (size=12493) 2024-11-23T15:41:57,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742209_1385 (size=4469) 2024-11-23T15:41:57,509 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/08723c521b354836852b0be87b93a24f as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/08723c521b354836852b0be87b93a24f 2024-11-23T15:41:57,515 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6680d3b6fc70270ea7a87351f7da4298/B of 6680d3b6fc70270ea7a87351f7da4298 into 08723c521b354836852b0be87b93a24f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:57,515 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6680d3b6fc70270ea7a87351f7da4298: 2024-11-23T15:41:57,515 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., storeName=6680d3b6fc70270ea7a87351f7da4298/B, priority=13, startTime=1732376517475; duration=0sec 2024-11-23T15:41:57,516 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:41:57,516 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6680d3b6fc70270ea7a87351f7da4298:B 2024-11-23T15:41:57,516 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:41:57,517 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:41:57,517 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): 6680d3b6fc70270ea7a87351f7da4298/C is initiating minor compaction (all files) 2024-11-23T15:41:57,517 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6680d3b6fc70270ea7a87351f7da4298/C in TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:57,517 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/e1ab255c73704ff6914020ed4db77531, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/150939c969044e8f83833fab71c1139f, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/05cae72f513a4fdca8190df178bc5e65] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp, totalSize=35.7 K 2024-11-23T15:41:57,517 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting e1ab255c73704ff6914020ed4db77531, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732376509715 2024-11-23T15:41:57,517 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 150939c969044e8f83833fab71c1139f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1732376512655 2024-11-23T15:41:57,518 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 05cae72f513a4fdca8190df178bc5e65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732376514658 2024-11-23T15:41:57,526 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6680d3b6fc70270ea7a87351f7da4298#C#compaction#329 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:57,526 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/c1224f810aa9419a99bdca72f95fc1f1 is 50, key is test_row_0/C:col10/1732376515817/Put/seqid=0 2024-11-23T15:41:57,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742210_1386 (size=12493) 2024-11-23T15:41:57,541 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/c1224f810aa9419a99bdca72f95fc1f1 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/c1224f810aa9419a99bdca72f95fc1f1 2024-11-23T15:41:57,545 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6680d3b6fc70270ea7a87351f7da4298/C of 6680d3b6fc70270ea7a87351f7da4298 into c1224f810aa9419a99bdca72f95fc1f1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:57,545 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6680d3b6fc70270ea7a87351f7da4298: 2024-11-23T15:41:57,545 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., storeName=6680d3b6fc70270ea7a87351f7da4298/C, priority=13, startTime=1732376517475; duration=0sec 2024-11-23T15:41:57,545 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:57,545 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6680d3b6fc70270ea7a87351f7da4298:C 2024-11-23T15:41:57,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-23T15:41:57,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:57,570 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 6680d3b6fc70270ea7a87351f7da4298 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-23T15:41:57,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123736b998fe97e42c6b43cae24813bc081_6680d3b6fc70270ea7a87351f7da4298 is 50, key is test_row_0/A:col10/1732376515871/Put/seqid=0 2024-11-23T15:41:57,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742211_1387 (size=12304) 2024-11-23T15:41:57,906 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6680d3b6fc70270ea7a87351f7da4298#A#compaction#328 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:41:57,907 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/ab386a11308e4fa78425d001f114a1c0 is 175, key is test_row_0/A:col10/1732376515817/Put/seqid=0 2024-11-23T15:41:57,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742212_1388 (size=31447) 2024-11-23T15:41:57,914 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/ab386a11308e4fa78425d001f114a1c0 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/ab386a11308e4fa78425d001f114a1c0 2024-11-23T15:41:57,918 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6680d3b6fc70270ea7a87351f7da4298/A of 6680d3b6fc70270ea7a87351f7da4298 into ab386a11308e4fa78425d001f114a1c0(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:41:57,919 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6680d3b6fc70270ea7a87351f7da4298: 2024-11-23T15:41:57,919 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., storeName=6680d3b6fc70270ea7a87351f7da4298/A, priority=13, startTime=1732376517474; duration=0sec 2024-11-23T15:41:57,919 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:41:57,919 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6680d3b6fc70270ea7a87351f7da4298:A 2024-11-23T15:41:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-23T15:41:57,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:57,982 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123736b998fe97e42c6b43cae24813bc081_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123736b998fe97e42c6b43cae24813bc081_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:57,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/2048a017d4aa46089dd95c81f87f9a1d, store: [table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:41:57,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/2048a017d4aa46089dd95c81f87f9a1d is 175, key is test_row_0/A:col10/1732376515871/Put/seqid=0 2024-11-23T15:41:57,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742213_1389 (size=31105) 2024-11-23T15:41:57,996 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=190, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/2048a017d4aa46089dd95c81f87f9a1d 2024-11-23T15:41:58,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/5533d0ff369f4be097a43edf0f791483 is 50, key is test_row_0/B:col10/1732376515871/Put/seqid=0 2024-11-23T15:41:58,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742214_1390 (size=12151) 2024-11-23T15:41:58,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:58,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:58,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:58,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376578218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:58,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:58,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:58,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:58,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376578324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:58,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:58,406 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/5533d0ff369f4be097a43edf0f791483 2024-11-23T15:41:58,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/c1dc065f4a0b4cb6b1ba4535cbee344b is 50, key is test_row_0/C:col10/1732376515871/Put/seqid=0 2024-11-23T15:41:58,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742215_1391 (size=12151) 2024-11-23T15:41:58,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376578534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:58,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:58,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:58,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:58,815 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/c1dc065f4a0b4cb6b1ba4535cbee344b 2024-11-23T15:41:58,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/2048a017d4aa46089dd95c81f87f9a1d as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/2048a017d4aa46089dd95c81f87f9a1d 2024-11-23T15:41:58,823 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/2048a017d4aa46089dd95c81f87f9a1d, entries=150, sequenceid=190, filesize=30.4 K 2024-11-23T15:41:58,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/5533d0ff369f4be097a43edf0f791483 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/5533d0ff369f4be097a43edf0f791483 2024-11-23T15:41:58,827 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/5533d0ff369f4be097a43edf0f791483, entries=150, sequenceid=190, filesize=11.9 K 2024-11-23T15:41:58,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/c1dc065f4a0b4cb6b1ba4535cbee344b as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/c1dc065f4a0b4cb6b1ba4535cbee344b 2024-11-23T15:41:58,831 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/c1dc065f4a0b4cb6b1ba4535cbee344b, entries=150, sequenceid=190, filesize=11.9 K 2024-11-23T15:41:58,832 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 6680d3b6fc70270ea7a87351f7da4298 in 1262ms, sequenceid=190, compaction requested=false 2024-11-23T15:41:58,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 6680d3b6fc70270ea7a87351f7da4298: 2024-11-23T15:41:58,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:58,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-23T15:41:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-23T15:41:58,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-23T15:41:58,835 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0260 sec 2024-11-23T15:41:58,836 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.0310 sec 2024-11-23T15:41:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:58,847 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:41:58,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6680d3b6fc70270ea7a87351f7da4298 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-23T15:41:58,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411234441b8efb56e431697a527d2d6669423_6680d3b6fc70270ea7a87351f7da4298 is 50, key is test_row_0/A:col10/1732376518216/Put/seqid=0 2024-11-23T15:41:58,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742216_1392 (size=14794) 2024-11-23T15:41:58,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-23T15:41:58,954 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:41:58,954 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:41:58,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-23T15:41:58,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-23T15:41:58,955 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:41:58,956 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:41:58,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:41:58,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:58,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376578968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:59,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-23T15:41:59,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:59,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376579074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:59,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:59,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:59,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:59,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-23T15:41:59,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:59,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:59,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:59,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:59,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:59,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:59,256 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:41:59,259 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411234441b8efb56e431697a527d2d6669423_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411234441b8efb56e431697a527d2d6669423_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:41:59,260 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/6030c5490736440f93fa13ec94b162b0, store: [table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:41:59,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/6030c5490736440f93fa13ec94b162b0 is 175, key is test_row_0/A:col10/1732376518216/Put/seqid=0 2024-11-23T15:41:59,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-23T15:41:59,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:59,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:59,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:59,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:59,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:59,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742217_1393 (size=39749) 2024-11-23T15:41:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-23T15:41:59,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376579284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:59,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:59,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:59,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:59,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-23T15:41:59,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:59,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:59,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:59,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:59,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:59,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-23T15:41:59,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:59,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:59,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:59,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:59,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:59,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-23T15:41:59,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:41:59,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376579593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:41:59,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:59,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:41:59,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:41:59,665 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/6030c5490736440f93fa13ec94b162b0 2024-11-23T15:41:59,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/7dd63d15af054787834cb22667e592b5 is 50, key is test_row_0/B:col10/1732376518216/Put/seqid=0 2024-11-23T15:41:59,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742218_1394 (size=12151) 2024-11-23T15:41:59,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-23T15:41:59,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:59,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:59,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:59,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:59,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:59,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:59,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-23T15:41:59,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:59,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:41:59,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:41:59,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:59,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:41:59,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:00,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-23T15:42:00,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:42:00,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:42:00,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:42:00,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:00,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:00,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:00,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/7dd63d15af054787834cb22667e592b5 2024-11-23T15:42:00,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/f9ffa86623fd4133a2ed19dccb6d8fd7 is 50, key is test_row_0/C:col10/1732376518216/Put/seqid=0 2024-11-23T15:42:00,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742219_1395 (size=12151) 2024-11-23T15:42:00,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-23T15:42:00,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:00,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376580104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:00,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:00,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-23T15:42:00,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:42:00,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:42:00,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:42:00,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:00,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:00,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:00,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-23T15:42:00,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:42:00,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:42:00,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:42:00,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:00,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:00,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/f9ffa86623fd4133a2ed19dccb6d8fd7 2024-11-23T15:42:00,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-23T15:42:00,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:42:00,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:42:00,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:42:00,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:00,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:00,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/6030c5490736440f93fa13ec94b162b0 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/6030c5490736440f93fa13ec94b162b0 2024-11-23T15:42:00,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/6030c5490736440f93fa13ec94b162b0, entries=200, sequenceid=206, filesize=38.8 K 2024-11-23T15:42:00,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/7dd63d15af054787834cb22667e592b5 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/7dd63d15af054787834cb22667e592b5 2024-11-23T15:42:00,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/7dd63d15af054787834cb22667e592b5, entries=150, sequenceid=206, filesize=11.9 K 2024-11-23T15:42:00,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/f9ffa86623fd4133a2ed19dccb6d8fd7 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/f9ffa86623fd4133a2ed19dccb6d8fd7 2024-11-23T15:42:00,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/f9ffa86623fd4133a2ed19dccb6d8fd7, entries=150, sequenceid=206, filesize=11.9 K 2024-11-23T15:42:00,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 6680d3b6fc70270ea7a87351f7da4298 in 1651ms, sequenceid=206, compaction requested=true 2024-11-23T15:42:00,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6680d3b6fc70270ea7a87351f7da4298: 2024-11-23T15:42:00,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6680d3b6fc70270ea7a87351f7da4298:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:42:00,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:00,498 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:00,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6680d3b6fc70270ea7a87351f7da4298:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:42:00,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:00,498 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:00,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6680d3b6fc70270ea7a87351f7da4298:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:42:00,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:00,499 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:00,499 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:00,499 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): 6680d3b6fc70270ea7a87351f7da4298/B is initiating minor compaction (all files) 2024-11-23T15:42:00,499 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): 6680d3b6fc70270ea7a87351f7da4298/A is initiating minor compaction (all files) 2024-11-23T15:42:00,499 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6680d3b6fc70270ea7a87351f7da4298/B in TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:42:00,499 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6680d3b6fc70270ea7a87351f7da4298/A in TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:42:00,499 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/08723c521b354836852b0be87b93a24f, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/5533d0ff369f4be097a43edf0f791483, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/7dd63d15af054787834cb22667e592b5] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp, totalSize=35.9 K 2024-11-23T15:42:00,499 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/ab386a11308e4fa78425d001f114a1c0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/2048a017d4aa46089dd95c81f87f9a1d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/6030c5490736440f93fa13ec94b162b0] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp, totalSize=99.9 K 2024-11-23T15:42:00,499 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:42:00,499 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. files: [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/ab386a11308e4fa78425d001f114a1c0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/2048a017d4aa46089dd95c81f87f9a1d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/6030c5490736440f93fa13ec94b162b0] 2024-11-23T15:42:00,499 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 08723c521b354836852b0be87b93a24f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732376514658 2024-11-23T15:42:00,499 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab386a11308e4fa78425d001f114a1c0, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732376514658 2024-11-23T15:42:00,499 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 5533d0ff369f4be097a43edf0f791483, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732376515871 2024-11-23T15:42:00,500 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2048a017d4aa46089dd95c81f87f9a1d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732376515871 2024-11-23T15:42:00,500 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dd63d15af054787834cb22667e592b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732376518209 2024-11-23T15:42:00,500 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6030c5490736440f93fa13ec94b162b0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732376518195 2024-11-23T15:42:00,504 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:42:00,506 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6680d3b6fc70270ea7a87351f7da4298#B#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:00,506 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/c82124275eee4a87a8023282ea768a1e is 50, key is test_row_0/B:col10/1732376518216/Put/seqid=0 2024-11-23T15:42:00,506 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411233e3f74d012a34e5b83ca1b112fb2d7ee_6680d3b6fc70270ea7a87351f7da4298 store=[table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:42:00,508 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411233e3f74d012a34e5b83ca1b112fb2d7ee_6680d3b6fc70270ea7a87351f7da4298, store=[table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:42:00,508 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411233e3f74d012a34e5b83ca1b112fb2d7ee_6680d3b6fc70270ea7a87351f7da4298 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:42:00,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742221_1397 (size=4469) 2024-11-23T15:42:00,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742220_1396 (size=12595) 2024-11-23T15:42:00,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-23T15:42:00,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:42:00,637 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 6680d3b6fc70270ea7a87351f7da4298 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-23T15:42:00,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123a78941bd42144219b69985364ce1234e_6680d3b6fc70270ea7a87351f7da4298 is 50, key is test_row_0/A:col10/1732376518956/Put/seqid=0 2024-11-23T15:42:00,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742222_1398 (size=12304) 2024-11-23T15:42:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:00,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. as already flushing 2024-11-23T15:42:00,912 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6680d3b6fc70270ea7a87351f7da4298#A#compaction#337 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:00,913 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/f8a99774f0854f70a3b5f4adf32cb0ee is 175, key is test_row_0/A:col10/1732376518216/Put/seqid=0 2024-11-23T15:42:00,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742223_1399 (size=31549) 2024-11-23T15:42:00,920 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/c82124275eee4a87a8023282ea768a1e as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/c82124275eee4a87a8023282ea768a1e 2024-11-23T15:42:00,924 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6680d3b6fc70270ea7a87351f7da4298/B of 6680d3b6fc70270ea7a87351f7da4298 into c82124275eee4a87a8023282ea768a1e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:00,924 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6680d3b6fc70270ea7a87351f7da4298: 2024-11-23T15:42:00,924 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., storeName=6680d3b6fc70270ea7a87351f7da4298/B, priority=13, startTime=1732376520498; duration=0sec 2024-11-23T15:42:00,924 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:00,924 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6680d3b6fc70270ea7a87351f7da4298:B 2024-11-23T15:42:00,924 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:00,925 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:00,925 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): 6680d3b6fc70270ea7a87351f7da4298/C is initiating minor compaction (all files) 2024-11-23T15:42:00,925 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6680d3b6fc70270ea7a87351f7da4298/C in TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:42:00,925 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/c1224f810aa9419a99bdca72f95fc1f1, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/c1dc065f4a0b4cb6b1ba4535cbee344b, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/f9ffa86623fd4133a2ed19dccb6d8fd7] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp, totalSize=35.9 K 2024-11-23T15:42:00,926 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting c1224f810aa9419a99bdca72f95fc1f1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732376514658 2024-11-23T15:42:00,926 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting c1dc065f4a0b4cb6b1ba4535cbee344b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732376515871 2024-11-23T15:42:00,926 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting f9ffa86623fd4133a2ed19dccb6d8fd7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732376518209 2024-11-23T15:42:00,933 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6680d3b6fc70270ea7a87351f7da4298#C#compaction#339 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:00,933 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/d250dd4ea9a24e7ca378e15dbc82156a is 50, key is test_row_0/C:col10/1732376518216/Put/seqid=0 2024-11-23T15:42:00,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742224_1400 (size=12595) 2024-11-23T15:42:00,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:00,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376580948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:00,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376580948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:00,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:00,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:00,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376580960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:00,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:00,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:00,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:00,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:00,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376580961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:00,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:00,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:00,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:01,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:01,056 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123a78941bd42144219b69985364ce1234e_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123a78941bd42144219b69985364ce1234e_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:01,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/ee56627f1f154bd68e922abfee027882, store: [table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:42:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/ee56627f1f154bd68e922abfee027882 is 175, key is test_row_0/A:col10/1732376518956/Put/seqid=0 2024-11-23T15:42:01,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742225_1401 (size=31105) 2024-11-23T15:42:01,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376581064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:01,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376581064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:01,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376581074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:01,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376581074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:01,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,081 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:01,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-23T15:42:01,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:01,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47222 deadline: 1732376581124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:01,130 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_2' in 6680d3b6fc70270ea7a87351f7da4298 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:42:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:01,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376581274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:01,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376581274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:01,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376581284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:01,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:01,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376581285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:01,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:01,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:01,320 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/f8a99774f0854f70a3b5f4adf32cb0ee as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/f8a99774f0854f70a3b5f4adf32cb0ee 2024-11-23T15:42:01,324 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6680d3b6fc70270ea7a87351f7da4298/A of 6680d3b6fc70270ea7a87351f7da4298 into f8a99774f0854f70a3b5f4adf32cb0ee(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:01,325 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6680d3b6fc70270ea7a87351f7da4298: 2024-11-23T15:42:01,325 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., storeName=6680d3b6fc70270ea7a87351f7da4298/A, priority=13, startTime=1732376520498; duration=0sec 2024-11-23T15:42:01,325 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:01,325 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6680d3b6fc70270ea7a87351f7da4298:A 2024-11-23T15:42:01,343 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/d250dd4ea9a24e7ca378e15dbc82156a as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/d250dd4ea9a24e7ca378e15dbc82156a 2024-11-23T15:42:01,347 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6680d3b6fc70270ea7a87351f7da4298/C of 6680d3b6fc70270ea7a87351f7da4298 into d250dd4ea9a24e7ca378e15dbc82156a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:01,347 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6680d3b6fc70270ea7a87351f7da4298: 2024-11-23T15:42:01,347 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., storeName=6680d3b6fc70270ea7a87351f7da4298/C, priority=13, startTime=1732376520498; duration=0sec 2024-11-23T15:42:01,347 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:01,347 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6680d3b6fc70270ea7a87351f7da4298:C 2024-11-23T15:42:01,461 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/ee56627f1f154bd68e922abfee027882 2024-11-23T15:42:01,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/94841079d00b46ef9c24754f0245de5e is 50, key is test_row_0/B:col10/1732376518956/Put/seqid=0 2024-11-23T15:42:01,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742226_1402 (size=12151) 2024-11-23T15:42:01,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376581584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376581584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:01,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376581594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376581594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:01,865 INFO [Thread-1576 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1576. 2024-11-23T15:42:01,865 DEBUG [Thread-1576 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:01,865 DEBUG [Thread-1576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:01,865 INFO [Thread-1572 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1572. 2024-11-23T15:42:01,865 DEBUG [Thread-1576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:01,865 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:01,865 DEBUG [Thread-1572 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:01,865 DEBUG [Thread-1572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:01,865 DEBUG [Thread-1572 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:01,865 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:01,870 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/94841079d00b46ef9c24754f0245de5e 2024-11-23T15:42:01,872 INFO [Thread-1574 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1574. 2024-11-23T15:42:01,872 DEBUG [Thread-1574 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:01,872 DEBUG [Thread-1574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:01,872 DEBUG [Thread-1574 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:01,872 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:01,873 INFO [Thread-1578 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1578. 2024-11-23T15:42:01,873 DEBUG [Thread-1578 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:01,873 DEBUG [Thread-1578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:01,873 DEBUG [Thread-1578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:01,873 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:01,873 INFO [Thread-1570 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1570. 2024-11-23T15:42:01,873 DEBUG [Thread-1570 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:01,873 DEBUG [Thread-1570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:01,873 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:01,873 DEBUG [Thread-1570 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:01,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/59a3a24d51864bf59ec8a79679ba408e is 50, key is test_row_0/C:col10/1732376518956/Put/seqid=0 2024-11-23T15:42:01,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742227_1403 (size=12151) 2024-11-23T15:42:02,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:02,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47278 deadline: 1732376582095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47252 deadline: 1732376582095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:02,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:02,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:02,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:02,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47220 deadline: 1732376582104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47224 deadline: 1732376582104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:02,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:02,282 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/59a3a24d51864bf59ec8a79679ba408e 2024-11-23T15:42:02,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/ee56627f1f154bd68e922abfee027882 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/ee56627f1f154bd68e922abfee027882 2024-11-23T15:42:02,296 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/ee56627f1f154bd68e922abfee027882, entries=150, sequenceid=227, filesize=30.4 K 2024-11-23T15:42:02,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/94841079d00b46ef9c24754f0245de5e as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/94841079d00b46ef9c24754f0245de5e 2024-11-23T15:42:02,304 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/94841079d00b46ef9c24754f0245de5e, entries=150, sequenceid=227, filesize=11.9 K 2024-11-23T15:42:02,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/59a3a24d51864bf59ec8a79679ba408e as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/59a3a24d51864bf59ec8a79679ba408e 2024-11-23T15:42:02,310 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/59a3a24d51864bf59ec8a79679ba408e, entries=150, sequenceid=227, filesize=11.9 K 2024-11-23T15:42:02,311 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 6680d3b6fc70270ea7a87351f7da4298 in 1674ms, sequenceid=227, compaction requested=false 2024-11-23T15:42:02,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 6680d3b6fc70270ea7a87351f7da4298: 2024-11-23T15:42:02,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:42:02,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-23T15:42:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-23T15:42:02,314 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-23T15:42:02,314 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3570 sec 2024-11-23T15:42:02,316 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 3.3610 sec 2024-11-23T15:42:03,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-23T15:42:03,115 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:03,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:03,117 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:03,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6680d3b6fc70270ea7a87351f7da4298 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-23T15:42:03,117 INFO [Thread-1567 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1567. 2024-11-23T15:42:03,117 INFO [Thread-1565 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1565. 2024-11-23T15:42:03,117 DEBUG [Thread-1567 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:03,117 DEBUG [Thread-1565 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:03,117 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:03,117 DEBUG [Thread-1567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:03,117 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:03,117 DEBUG [Thread-1567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:03,117 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:03,117 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:03,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112388374063fc214b0eb3dfb0c7cc0bcfd5_6680d3b6fc70270ea7a87351f7da4298 is 50, key is test_row_0/A:col10/1732376520946/Put/seqid=0 2024-11-23T15:42:03,126 INFO [Thread-1563 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1563. 2024-11-23T15:42:03,126 DEBUG [Thread-1563 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:03,126 DEBUG [Thread-1563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:03,126 INFO [Thread-1561 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1561. 2024-11-23T15:42:03,126 DEBUG [Thread-1561 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:03,127 DEBUG [Thread-1561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:03,127 DEBUG [Thread-1563 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:03,127 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:03,127 DEBUG [Thread-1561 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:03,127 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:03,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742228_1404 (size=12304) 2024-11-23T15:42:03,136 INFO [Thread-1559 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1559. 2024-11-23T15:42:03,136 DEBUG [Thread-1559 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:03,136 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:03,136 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:03,136 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:03,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T15:42:03,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-23T15:42:03,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-23T15:42:03,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-23T15:42:03,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-23T15:42:03,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-23T15:42:03,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T15:42:03,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T15:42:03,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1492 2024-11-23T15:42:03,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4476 rows 2024-11-23T15:42:03,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1486 2024-11-23T15:42:03,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4457 rows 2024-11-23T15:42:03,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1486 2024-11-23T15:42:03,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4458 rows 2024-11-23T15:42:03,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1490 2024-11-23T15:42:03,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4465 rows 2024-11-23T15:42:03,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1488 2024-11-23T15:42:03,136 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4459 rows 2024-11-23T15:42:03,136 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T15:42:03,137 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T15:42:03,137 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:03,137 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:03,137 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:03,137 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-23T15:42:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T15:42:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-23T15:42:03,140 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376523140"}]},"ts":"1732376523140"} 2024-11-23T15:42:03,141 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T15:42:03,141 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T15:42:03,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T15:42:03,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6680d3b6fc70270ea7a87351f7da4298, UNASSIGN}] 2024-11-23T15:42:03,144 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6680d3b6fc70270ea7a87351f7da4298, UNASSIGN 2024-11-23T15:42:03,145 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=6680d3b6fc70270ea7a87351f7da4298, regionState=CLOSING, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:03,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6680d3b6fc70270ea7a87351f7da4298, UNASSIGN because future has completed 2024-11-23T15:42:03,146 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T15:42:03,146 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194}] 2024-11-23T15:42:03,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-23T15:42:03,298 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(122): Close 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:03,298 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T15:42:03,298 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1722): Closing 6680d3b6fc70270ea7a87351f7da4298, disabling compactions & flushes 2024-11-23T15:42:03,298 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:42:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-23T15:42:03,530 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:03,533 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112388374063fc214b0eb3dfb0c7cc0bcfd5_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112388374063fc214b0eb3dfb0c7cc0bcfd5_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:03,534 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/7ac709484cfc4f49a479b75d1e4f395b, store: [table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:42:03,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/7ac709484cfc4f49a479b75d1e4f395b is 175, key is test_row_0/A:col10/1732376520946/Put/seqid=0 2024-11-23T15:42:03,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742229_1405 (size=31105) 2024-11-23T15:42:03,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-23T15:42:03,939 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/7ac709484cfc4f49a479b75d1e4f395b 2024-11-23T15:42:03,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/0e104cc0696d4ac7a9a1763e6b89e977 is 50, key is test_row_0/B:col10/1732376520946/Put/seqid=0 2024-11-23T15:42:03,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742230_1406 (size=12151) 2024-11-23T15:42:04,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-23T15:42:04,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/0e104cc0696d4ac7a9a1763e6b89e977 2024-11-23T15:42:04,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/4eb075f6baf847a88d9a9cae9787f906 is 50, key is test_row_0/C:col10/1732376520946/Put/seqid=0 2024-11-23T15:42:04,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742231_1407 (size=12151) 2024-11-23T15:42:04,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/4eb075f6baf847a88d9a9cae9787f906 2024-11-23T15:42:04,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/7ac709484cfc4f49a479b75d1e4f395b as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/7ac709484cfc4f49a479b75d1e4f395b 2024-11-23T15:42:04,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/7ac709484cfc4f49a479b75d1e4f395b, entries=150, sequenceid=248, filesize=30.4 K 2024-11-23T15:42:04,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/0e104cc0696d4ac7a9a1763e6b89e977 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/0e104cc0696d4ac7a9a1763e6b89e977 2024-11-23T15:42:04,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/0e104cc0696d4ac7a9a1763e6b89e977, entries=150, sequenceid=248, filesize=11.9 K 2024-11-23T15:42:04,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/4eb075f6baf847a88d9a9cae9787f906 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/4eb075f6baf847a88d9a9cae9787f906 2024-11-23T15:42:04,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/4eb075f6baf847a88d9a9cae9787f906, entries=150, sequenceid=248, filesize=11.9 K 2024-11-23T15:42:04,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=20.13 KB/20610 for 6680d3b6fc70270ea7a87351f7da4298 in 1678ms, sequenceid=248, compaction requested=true 2024-11-23T15:42:04,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6680d3b6fc70270ea7a87351f7da4298: 2024-11-23T15:42:04,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6680d3b6fc70270ea7a87351f7da4298:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:42:04,795 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:42:04,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:04,796 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. because compaction request was cancelled 2024-11-23T15:42:04,796 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:42:04,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6680d3b6fc70270ea7a87351f7da4298:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:42:04,796 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6680d3b6fc70270ea7a87351f7da4298:A 2024-11-23T15:42:04,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:04,796 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. after waiting 0 ms 2024-11-23T15:42:04,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6680d3b6fc70270ea7a87351f7da4298:C, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:42:04,796 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. because compaction request was cancelled 2024-11-23T15:42:04,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:04,796 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6680d3b6fc70270ea7a87351f7da4298:B 2024-11-23T15:42:04,796 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. because compaction request was cancelled 2024-11-23T15:42:04,796 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:42:04,796 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6680d3b6fc70270ea7a87351f7da4298:C 2024-11-23T15:42:04,796 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(2902): Flushing 6680d3b6fc70270ea7a87351f7da4298 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-23T15:42:04,803 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411230106c50db452433b84657bb404a9bedd_6680d3b6fc70270ea7a87351f7da4298 is 50, key is test_row_0/A:col10/1732376523125/Put/seqid=0 2024-11-23T15:42:04,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742232_1408 (size=9814) 2024-11-23T15:42:05,208 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:05,215 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411230106c50db452433b84657bb404a9bedd_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230106c50db452433b84657bb404a9bedd_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:05,217 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/27c60dbbe27a440597e4b1d268e7a06c, store: [table=TestAcidGuarantees family=A region=6680d3b6fc70270ea7a87351f7da4298] 2024-11-23T15:42:05,218 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/27c60dbbe27a440597e4b1d268e7a06c is 175, key is test_row_0/A:col10/1732376523125/Put/seqid=0 2024-11-23T15:42:05,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742233_1409 (size=22461) 2024-11-23T15:42:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-23T15:42:05,626 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/27c60dbbe27a440597e4b1d268e7a06c 2024-11-23T15:42:05,640 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/385c73733a0b44169f93483b097aed2a is 50, key is test_row_0/B:col10/1732376523125/Put/seqid=0 2024-11-23T15:42:05,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742234_1410 (size=9757) 2024-11-23T15:42:06,044 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/385c73733a0b44169f93483b097aed2a 2024-11-23T15:42:06,052 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/b3d9f60a74e04b05878eb598fcff2a18 is 50, key is test_row_0/C:col10/1732376523125/Put/seqid=0 2024-11-23T15:42:06,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742235_1411 (size=9757) 2024-11-23T15:42:06,457 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/b3d9f60a74e04b05878eb598fcff2a18 2024-11-23T15:42:06,461 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/A/27c60dbbe27a440597e4b1d268e7a06c as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/27c60dbbe27a440597e4b1d268e7a06c 2024-11-23T15:42:06,464 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/27c60dbbe27a440597e4b1d268e7a06c, entries=100, sequenceid=254, filesize=21.9 K 2024-11-23T15:42:06,465 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/B/385c73733a0b44169f93483b097aed2a as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/385c73733a0b44169f93483b097aed2a 2024-11-23T15:42:06,468 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/385c73733a0b44169f93483b097aed2a, entries=100, sequenceid=254, filesize=9.5 K 2024-11-23T15:42:06,469 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/.tmp/C/b3d9f60a74e04b05878eb598fcff2a18 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/b3d9f60a74e04b05878eb598fcff2a18 2024-11-23T15:42:06,473 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/b3d9f60a74e04b05878eb598fcff2a18, entries=100, sequenceid=254, filesize=9.5 K 2024-11-23T15:42:06,473 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 6680d3b6fc70270ea7a87351f7da4298 in 1677ms, sequenceid=254, compaction requested=true 2024-11-23T15:42:06,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/9f56b2d9097140a091a2f01adb92df3c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/5d18c4e0d8a94a56b6ab0e5e87780ef8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/2e876db9b873453ba65376d35201a6df, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/2ced121e989f4deab318159e8dad2cdf, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/d5db65e606da4ab083fe11d648ea277c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/3440e7bd369d47bc8cec4004e6a18a81, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/1905e303e55149feaa67d7666c2636e2, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/99702eeab3f6416dad1602d8b105f580, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/3c70e8ae27074d9595e95a314581b769, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/dc9b3c33f6d34b16bd095325c8c63452, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/ab386a11308e4fa78425d001f114a1c0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/2048a017d4aa46089dd95c81f87f9a1d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/6030c5490736440f93fa13ec94b162b0] to archive 2024-11-23T15:42:06,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T15:42:06,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/9f56b2d9097140a091a2f01adb92df3c to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/9f56b2d9097140a091a2f01adb92df3c 2024-11-23T15:42:06,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/5d18c4e0d8a94a56b6ab0e5e87780ef8 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/5d18c4e0d8a94a56b6ab0e5e87780ef8 2024-11-23T15:42:06,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/2e876db9b873453ba65376d35201a6df to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/2e876db9b873453ba65376d35201a6df 2024-11-23T15:42:06,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/2ced121e989f4deab318159e8dad2cdf to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/2ced121e989f4deab318159e8dad2cdf 2024-11-23T15:42:06,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/d5db65e606da4ab083fe11d648ea277c to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/d5db65e606da4ab083fe11d648ea277c 2024-11-23T15:42:06,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/3440e7bd369d47bc8cec4004e6a18a81 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/3440e7bd369d47bc8cec4004e6a18a81 2024-11-23T15:42:06,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/1905e303e55149feaa67d7666c2636e2 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/1905e303e55149feaa67d7666c2636e2 2024-11-23T15:42:06,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/99702eeab3f6416dad1602d8b105f580 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/99702eeab3f6416dad1602d8b105f580 2024-11-23T15:42:06,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/3c70e8ae27074d9595e95a314581b769 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/3c70e8ae27074d9595e95a314581b769 2024-11-23T15:42:06,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/dc9b3c33f6d34b16bd095325c8c63452 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/dc9b3c33f6d34b16bd095325c8c63452 2024-11-23T15:42:06,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/ab386a11308e4fa78425d001f114a1c0 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/ab386a11308e4fa78425d001f114a1c0 2024-11-23T15:42:06,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/2048a017d4aa46089dd95c81f87f9a1d to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/2048a017d4aa46089dd95c81f87f9a1d 2024-11-23T15:42:06,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/6030c5490736440f93fa13ec94b162b0 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/6030c5490736440f93fa13ec94b162b0 2024-11-23T15:42:06,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/0e457f9baa504bcd8b055fdfa40a4977, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/a38db9c3c70042bc99df257bc503a374, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/54c9678b6de946f8a2fe34ff95b05825, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/e4c5a7b081ee4ba5992301c34e83436f, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/017d466bd9f64fc38b2e2542c1556392, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/d7e41a3e24834da8b45449462c3b16a6, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/d0502014413745d1bb8d5f93267d159e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/52b8533475a244f6beabf7322ba161bc, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/7caa71e684294ee6ac85580b450de695, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/08723c521b354836852b0be87b93a24f, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/73cf4b5df06c481bb5b3cae88d724c3c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/5533d0ff369f4be097a43edf0f791483, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/7dd63d15af054787834cb22667e592b5] to archive 2024-11-23T15:42:06,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T15:42:06,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/0e457f9baa504bcd8b055fdfa40a4977 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/0e457f9baa504bcd8b055fdfa40a4977 2024-11-23T15:42:06,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/a38db9c3c70042bc99df257bc503a374 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/a38db9c3c70042bc99df257bc503a374 2024-11-23T15:42:06,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/54c9678b6de946f8a2fe34ff95b05825 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/54c9678b6de946f8a2fe34ff95b05825 2024-11-23T15:42:06,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/e4c5a7b081ee4ba5992301c34e83436f to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/e4c5a7b081ee4ba5992301c34e83436f 2024-11-23T15:42:06,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/017d466bd9f64fc38b2e2542c1556392 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/017d466bd9f64fc38b2e2542c1556392 2024-11-23T15:42:06,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/d7e41a3e24834da8b45449462c3b16a6 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/d7e41a3e24834da8b45449462c3b16a6 2024-11-23T15:42:06,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/d0502014413745d1bb8d5f93267d159e to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/d0502014413745d1bb8d5f93267d159e 2024-11-23T15:42:06,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/52b8533475a244f6beabf7322ba161bc to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/52b8533475a244f6beabf7322ba161bc 2024-11-23T15:42:06,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/7caa71e684294ee6ac85580b450de695 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/7caa71e684294ee6ac85580b450de695 2024-11-23T15:42:06,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/08723c521b354836852b0be87b93a24f to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/08723c521b354836852b0be87b93a24f 2024-11-23T15:42:06,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/73cf4b5df06c481bb5b3cae88d724c3c to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/73cf4b5df06c481bb5b3cae88d724c3c 2024-11-23T15:42:06,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/5533d0ff369f4be097a43edf0f791483 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/5533d0ff369f4be097a43edf0f791483 2024-11-23T15:42:06,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/7dd63d15af054787834cb22667e592b5 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/7dd63d15af054787834cb22667e592b5 2024-11-23T15:42:06,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/47bf541253e14ca7abb60e39b4ededed, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/863fd9a825cb41c081f6a82e625696bb, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/7218910670354257aea83ae9fbfeb5fb, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/655855cfa4cd4243a871ef4d0e99588d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/4add009d8eb24a8e819dde3c36922c12, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/5b5dff6e86a14c67bfc94435e363d45b, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/e1ab255c73704ff6914020ed4db77531, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/a687b7736cc8484ea0143d15d9d0951a, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/150939c969044e8f83833fab71c1139f, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/c1224f810aa9419a99bdca72f95fc1f1, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/05cae72f513a4fdca8190df178bc5e65, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/c1dc065f4a0b4cb6b1ba4535cbee344b, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/f9ffa86623fd4133a2ed19dccb6d8fd7] to archive 2024-11-23T15:42:06,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T15:42:06,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/47bf541253e14ca7abb60e39b4ededed to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/47bf541253e14ca7abb60e39b4ededed 2024-11-23T15:42:06,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/863fd9a825cb41c081f6a82e625696bb to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/863fd9a825cb41c081f6a82e625696bb 2024-11-23T15:42:06,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/7218910670354257aea83ae9fbfeb5fb to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/7218910670354257aea83ae9fbfeb5fb 2024-11-23T15:42:06,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/655855cfa4cd4243a871ef4d0e99588d to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/655855cfa4cd4243a871ef4d0e99588d 2024-11-23T15:42:06,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/4add009d8eb24a8e819dde3c36922c12 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/4add009d8eb24a8e819dde3c36922c12 2024-11-23T15:42:06,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/5b5dff6e86a14c67bfc94435e363d45b to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/5b5dff6e86a14c67bfc94435e363d45b 2024-11-23T15:42:06,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/e1ab255c73704ff6914020ed4db77531 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/e1ab255c73704ff6914020ed4db77531 2024-11-23T15:42:06,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/a687b7736cc8484ea0143d15d9d0951a to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/a687b7736cc8484ea0143d15d9d0951a 2024-11-23T15:42:06,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/150939c969044e8f83833fab71c1139f to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/150939c969044e8f83833fab71c1139f 2024-11-23T15:42:06,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/c1224f810aa9419a99bdca72f95fc1f1 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/c1224f810aa9419a99bdca72f95fc1f1 2024-11-23T15:42:06,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/05cae72f513a4fdca8190df178bc5e65 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/05cae72f513a4fdca8190df178bc5e65 2024-11-23T15:42:06,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/c1dc065f4a0b4cb6b1ba4535cbee344b to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/c1dc065f4a0b4cb6b1ba4535cbee344b 2024-11-23T15:42:06,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/f9ffa86623fd4133a2ed19dccb6d8fd7 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/f9ffa86623fd4133a2ed19dccb6d8fd7 2024-11-23T15:42:06,512 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/recovered.edits/257.seqid, newMaxSeqId=257, maxSeqId=4 2024-11-23T15:42:06,513 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. 2024-11-23T15:42:06,513 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1676): Region close journal for 6680d3b6fc70270ea7a87351f7da4298: Waiting for close lock at 1732376523298Running coprocessor pre-close hooks at 1732376523298Disabling compacts and flushes for region at 1732376523298Disabling writes for close at 1732376524796 (+1498 ms)Obtaining lock to block concurrent updates at 1732376524796Preparing flush snapshotting stores in 6680d3b6fc70270ea7a87351f7da4298 at 1732376524796Finished memstore snapshotting TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732376524796Flushing stores of TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298. at 1732376524797 (+1 ms)Flushing 6680d3b6fc70270ea7a87351f7da4298/A: creating writer at 1732376524797Flushing mob file 6680d3b6fc70270ea7a87351f7da4298/A: appending metadata at 1732376524803 (+6 ms)Flushing mob file 6680d3b6fc70270ea7a87351f7da4298/A: closing flushed file at 1732376524803Flushing 6680d3b6fc70270ea7a87351f7da4298/A: appending metadata at 1732376525217 (+414 ms)Flushing 6680d3b6fc70270ea7a87351f7da4298/A: closing flushed file at 1732376525217Flushing 6680d3b6fc70270ea7a87351f7da4298/B: creating writer at 1732376525634 (+417 ms)Flushing 6680d3b6fc70270ea7a87351f7da4298/B: appending metadata at 1732376525639 (+5 ms)Flushing 6680d3b6fc70270ea7a87351f7da4298/B: closing flushed file at 1732376525639Flushing 6680d3b6fc70270ea7a87351f7da4298/C: creating writer at 1732376526049 (+410 ms)Flushing 6680d3b6fc70270ea7a87351f7da4298/C: appending metadata at 1732376526052 (+3 ms)Flushing 6680d3b6fc70270ea7a87351f7da4298/C: closing flushed file at 1732376526052Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76866a46: reopening flushed file at 1732376526460 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32d66fd2: reopening flushed file at 1732376526464 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b3ba107: reopening flushed file at 1732376526468 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 6680d3b6fc70270ea7a87351f7da4298 in 1677ms, sequenceid=254, compaction requested=true at 1732376526474 (+6 ms)Writing region close event to WAL at 1732376526510 (+36 ms)Running coprocessor post-close hooks at 1732376526512 (+2 ms)Closed at 1732376526513 (+1 ms) 2024-11-23T15:42:06,514 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(157): Closed 6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:06,515 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=6680d3b6fc70270ea7a87351f7da4298, regionState=CLOSED 2024-11-23T15:42:06,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 because future has completed 2024-11-23T15:42:06,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-23T15:42:06,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseRegionProcedure 6680d3b6fc70270ea7a87351f7da4298, server=b712f9af2c12,46589,1732376409194 in 3.3700 sec 2024-11-23T15:42:06,519 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-23T15:42:06,519 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6680d3b6fc70270ea7a87351f7da4298, UNASSIGN in 3.3750 sec 2024-11-23T15:42:06,520 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-23T15:42:06,520 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.3770 sec 2024-11-23T15:42:06,521 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376526521"}]},"ts":"1732376526521"} 2024-11-23T15:42:06,522 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T15:42:06,522 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T15:42:06,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.3860 sec 2024-11-23T15:42:07,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-23T15:42:07,295 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:07,296 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-23T15:42:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:42:07,301 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:42:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-23T15:42:07,303 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=123, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:42:07,307 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:07,310 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A, FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B, FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C, FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/recovered.edits] 2024-11-23T15:42:07,314 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/27c60dbbe27a440597e4b1d268e7a06c to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/27c60dbbe27a440597e4b1d268e7a06c 2024-11-23T15:42:07,316 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/7ac709484cfc4f49a479b75d1e4f395b to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/7ac709484cfc4f49a479b75d1e4f395b 2024-11-23T15:42:07,318 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/ee56627f1f154bd68e922abfee027882 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/ee56627f1f154bd68e922abfee027882 2024-11-23T15:42:07,319 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/f8a99774f0854f70a3b5f4adf32cb0ee to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/A/f8a99774f0854f70a3b5f4adf32cb0ee 2024-11-23T15:42:07,321 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/0e104cc0696d4ac7a9a1763e6b89e977 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/0e104cc0696d4ac7a9a1763e6b89e977 2024-11-23T15:42:07,323 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/385c73733a0b44169f93483b097aed2a to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/385c73733a0b44169f93483b097aed2a 2024-11-23T15:42:07,324 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/94841079d00b46ef9c24754f0245de5e to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/94841079d00b46ef9c24754f0245de5e 2024-11-23T15:42:07,325 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/c82124275eee4a87a8023282ea768a1e to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/B/c82124275eee4a87a8023282ea768a1e 2024-11-23T15:42:07,327 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/4eb075f6baf847a88d9a9cae9787f906 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/4eb075f6baf847a88d9a9cae9787f906 2024-11-23T15:42:07,328 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/59a3a24d51864bf59ec8a79679ba408e to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/59a3a24d51864bf59ec8a79679ba408e 2024-11-23T15:42:07,329 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/b3d9f60a74e04b05878eb598fcff2a18 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/b3d9f60a74e04b05878eb598fcff2a18 2024-11-23T15:42:07,330 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/d250dd4ea9a24e7ca378e15dbc82156a to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/C/d250dd4ea9a24e7ca378e15dbc82156a 2024-11-23T15:42:07,333 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/recovered.edits/257.seqid to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298/recovered.edits/257.seqid 2024-11-23T15:42:07,334 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:07,334 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T15:42:07,334 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T15:42:07,335 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-23T15:42:07,338 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230106c50db452433b84657bb404a9bedd_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230106c50db452433b84657bb404a9bedd_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:07,339 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112317cbfab30e344c3f8c59b5a58c55af72_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112317cbfab30e344c3f8c59b5a58c55af72_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:07,341 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123200e331e12b048bd933a5e2bda411014_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123200e331e12b048bd933a5e2bda411014_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:07,342 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112341f75c3f9d094e0690b76e20f7fd119f_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112341f75c3f9d094e0690b76e20f7fd119f_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:07,343 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411234441b8efb56e431697a527d2d6669423_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411234441b8efb56e431697a527d2d6669423_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:07,345 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123527976d74f314486babd4d3350a1d661_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123527976d74f314486babd4d3350a1d661_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:07,346 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236bfe8e65c8dc45ac9df9ea9a95e530f5_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236bfe8e65c8dc45ac9df9ea9a95e530f5_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:07,348 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123736b998fe97e42c6b43cae24813bc081_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123736b998fe97e42c6b43cae24813bc081_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:07,349 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112388374063fc214b0eb3dfb0c7cc0bcfd5_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112388374063fc214b0eb3dfb0c7cc0bcfd5_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:07,351 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123899281f6b50c456f95833ea2d6206bbd_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123899281f6b50c456f95833ea2d6206bbd_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:07,351 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123a78941bd42144219b69985364ce1234e_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123a78941bd42144219b69985364ce1234e_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:07,352 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123cd9163fb33dd442093b0dafe13f457e1_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123cd9163fb33dd442093b0dafe13f457e1_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:07,353 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e1d8b3041f1d43eca1de9d4739021ada_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e1d8b3041f1d43eca1de9d4739021ada_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:07,354 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123f082dcfe72204c80a891dc74faf3af4e_6680d3b6fc70270ea7a87351f7da4298 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123f082dcfe72204c80a891dc74faf3af4e_6680d3b6fc70270ea7a87351f7da4298 2024-11-23T15:42:07,354 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T15:42:07,356 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=123, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:42:07,358 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T15:42:07,359 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T15:42:07,360 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=123, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:42:07,360 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T15:42:07,360 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732376527360"}]},"ts":"9223372036854775807"} 2024-11-23T15:42:07,362 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T15:42:07,362 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6680d3b6fc70270ea7a87351f7da4298, NAME => 'TestAcidGuarantees,,1732376498702.6680d3b6fc70270ea7a87351f7da4298.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T15:42:07,362 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T15:42:07,362 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732376527362"}]},"ts":"9223372036854775807"} 2024-11-23T15:42:07,363 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T15:42:07,364 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=123, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:42:07,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 67 msec 2024-11-23T15:42:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-23T15:42:07,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T15:42:07,417 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:07,428 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240 (was 240), OpenFileDescriptor=457 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=393 (was 478), ProcessCount=11 (was 11), AvailableMemoryMB=7681 (was 7715) 2024-11-23T15:42:07,436 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=393, ProcessCount=11, AvailableMemoryMB=7681 2024-11-23T15:42:07,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T15:42:07,437 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:42:07,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T15:42:07,439 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T15:42:07,439 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:07,439 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 124 2024-11-23T15:42:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-23T15:42:07,439 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T15:42:07,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742236_1412 (size=959) 2024-11-23T15:42:07,451 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T15:42:07,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-23T15:42:07,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-23T15:42:07,850 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e 2024-11-23T15:42:07,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742237_1413 (size=53) 2024-11-23T15:42:08,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-23T15:42:08,261 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:42:08,261 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2888821c5754525ea6530da454808a84, disabling compactions & flushes 2024-11-23T15:42:08,261 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:08,261 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:08,261 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. after waiting 0 ms 2024-11-23T15:42:08,262 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:08,262 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:08,262 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2888821c5754525ea6530da454808a84: Waiting for close lock at 1732376528261Disabling compacts and flushes for region at 1732376528261Disabling writes for close at 1732376528261Writing region close event to WAL at 1732376528262 (+1 ms)Closed at 1732376528262 2024-11-23T15:42:08,265 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T15:42:08,265 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732376528265"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732376528265"}]},"ts":"1732376528265"} 2024-11-23T15:42:08,267 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T15:42:08,268 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T15:42:08,268 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376528268"}]},"ts":"1732376528268"} 2024-11-23T15:42:08,269 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T15:42:08,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2888821c5754525ea6530da454808a84, ASSIGN}] 2024-11-23T15:42:08,270 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2888821c5754525ea6530da454808a84, ASSIGN 2024-11-23T15:42:08,271 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2888821c5754525ea6530da454808a84, ASSIGN; state=OFFLINE, location=b712f9af2c12,46589,1732376409194; forceNewPlan=false, retain=false 2024-11-23T15:42:08,422 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=2888821c5754525ea6530da454808a84, regionState=OPENING, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:08,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2888821c5754525ea6530da454808a84, ASSIGN because future has completed 2024-11-23T15:42:08,423 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194}] 2024-11-23T15:42:08,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-23T15:42:08,580 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:08,580 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7752): Opening region: {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:42:08,583 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2888821c5754525ea6530da454808a84 2024-11-23T15:42:08,583 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:42:08,583 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7794): checking encryption for 2888821c5754525ea6530da454808a84 2024-11-23T15:42:08,583 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7797): checking classloading for 2888821c5754525ea6530da454808a84 2024-11-23T15:42:08,585 INFO [StoreOpener-2888821c5754525ea6530da454808a84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2888821c5754525ea6530da454808a84 2024-11-23T15:42:08,586 INFO [StoreOpener-2888821c5754525ea6530da454808a84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2888821c5754525ea6530da454808a84 columnFamilyName A 2024-11-23T15:42:08,587 DEBUG [StoreOpener-2888821c5754525ea6530da454808a84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:08,587 INFO [StoreOpener-2888821c5754525ea6530da454808a84-1 {}] regionserver.HStore(327): Store=2888821c5754525ea6530da454808a84/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:42:08,587 INFO [StoreOpener-2888821c5754525ea6530da454808a84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2888821c5754525ea6530da454808a84 2024-11-23T15:42:08,588 INFO [StoreOpener-2888821c5754525ea6530da454808a84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2888821c5754525ea6530da454808a84 columnFamilyName B 2024-11-23T15:42:08,588 DEBUG [StoreOpener-2888821c5754525ea6530da454808a84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:08,588 INFO [StoreOpener-2888821c5754525ea6530da454808a84-1 {}] regionserver.HStore(327): Store=2888821c5754525ea6530da454808a84/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:42:08,589 INFO [StoreOpener-2888821c5754525ea6530da454808a84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2888821c5754525ea6530da454808a84 2024-11-23T15:42:08,589 INFO [StoreOpener-2888821c5754525ea6530da454808a84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2888821c5754525ea6530da454808a84 columnFamilyName C 2024-11-23T15:42:08,590 DEBUG [StoreOpener-2888821c5754525ea6530da454808a84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:08,590 INFO [StoreOpener-2888821c5754525ea6530da454808a84-1 {}] regionserver.HStore(327): Store=2888821c5754525ea6530da454808a84/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:42:08,590 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1038): replaying wal for 2888821c5754525ea6530da454808a84 2024-11-23T15:42:08,591 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84 2024-11-23T15:42:08,591 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84 2024-11-23T15:42:08,592 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1048): stopping wal replay for 2888821c5754525ea6530da454808a84 2024-11-23T15:42:08,592 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1060): Cleaning up temporary data for 2888821c5754525ea6530da454808a84 2024-11-23T15:42:08,592 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T15:42:08,594 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): writing seq id for 2888821c5754525ea6530da454808a84 2024-11-23T15:42:08,595 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:42:08,596 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1114): Opened 2888821c5754525ea6530da454808a84; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69511804, jitterRate=0.03580659627914429}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T15:42:08,596 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2888821c5754525ea6530da454808a84 2024-11-23T15:42:08,596 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1006): Region open journal for 2888821c5754525ea6530da454808a84: Running coprocessor pre-open hook at 1732376528583Writing region info on filesystem at 1732376528583Initializing all the Stores at 1732376528584 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376528584Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376528585 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376528585Cleaning up temporary data from old regions at 1732376528592 (+7 ms)Running coprocessor post-open hooks at 1732376528596 (+4 ms)Region opened successfully at 1732376528596 2024-11-23T15:42:08,597 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., pid=126, masterSystemTime=1732376528574 2024-11-23T15:42:08,599 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:08,599 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:08,599 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=2888821c5754525ea6530da454808a84, regionState=OPEN, openSeqNum=2, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:08,600 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 because future has completed 2024-11-23T15:42:08,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-23T15:42:08,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; OpenRegionProcedure 2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 in 178 msec 2024-11-23T15:42:08,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-23T15:42:08,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2888821c5754525ea6530da454808a84, ASSIGN in 333 msec 2024-11-23T15:42:08,604 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T15:42:08,604 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376528604"}]},"ts":"1732376528604"} 2024-11-23T15:42:08,605 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T15:42:08,606 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T15:42:08,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1690 sec 2024-11-23T15:42:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-23T15:42:09,585 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:09,587 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d1f058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,587 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:42:09,587 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:42:09,589 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:42:09,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:42:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:42:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba8fd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:42:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:42:09,591 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:09,592 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:42:09,593 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e2e6e6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,594 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:42:09,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:42:09,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:42:09,598 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:42:09,599 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T15:42:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is b712f9af2c12,42351,1732376408377 2024-11-23T15:42:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1359031e 2024-11-23T15:42:09,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T15:42:09,601 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T15:42:09,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,603 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:42:09,604 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:42:09,604 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:42:09,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:42:09,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:42:09,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1252bbb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:42:09,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:42:09,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:09,606 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:42:09,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1db79b63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,607 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,607 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:42:09,607 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:42:09,607 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:42:09,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:42:09,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:42:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69a72839, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:42:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:42:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:09,608 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:42:09,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25ce6a4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,609 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:42:09,609 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:42:09,610 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:42:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:42:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:42:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ccf1900, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:42:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:42:09,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:09,611 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:42:09,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b0240ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,612 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:42:09,612 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:42:09,612 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:42:09,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:42:09,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:42:09,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e4cbd8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:42:09,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:42:09,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:09,613 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42084, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:42:09,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d99cd7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,614 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,614 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:42:09,614 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:42:09,614 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:42:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:42:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:42:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19cb48af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:42:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:42:09,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:09,616 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:42:09,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f8e4adc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,616 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,616 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:42:09,617 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:42:09,617 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:42:09,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:42:09,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:42:09,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46c837e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:42:09,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:42:09,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:09,618 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:42:09,619 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@351f0d45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,619 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:42:09,619 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:42:09,620 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:42:09,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:42:09,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:42:09,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@538a8ceb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:42:09,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:42:09,621 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:09,621 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:42:09,622 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4af7e713, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,622 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,622 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:42:09,623 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:42:09,623 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:42:09,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:42:09,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:42:09,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@179972dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,623 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:42:09,624 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:42:09,624 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:09,624 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:42:09,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ad93df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,625 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:42:09,625 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:42:09,626 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:42:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:42:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:42:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ecf148, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:42:09,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:42:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:09,627 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:42:09,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64a13a61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,628 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:42:09,628 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:42:09,629 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:42:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:42:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:42:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dff484b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:42:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:42:09,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:09,630 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:42:09,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79f3e649, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:09,635 DEBUG [Thread-1878 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:42:09,635 DEBUG [Thread-1867 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:42:09,635 DEBUG [Thread-1886 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:42:09,635 DEBUG [Thread-1869 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:42:09,635 DEBUG [Thread-1875 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:42:09,635 DEBUG [Thread-1880 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:42:09,635 DEBUG [Thread-1873 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:42:09,635 DEBUG [Thread-1882 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:42:09,636 DEBUG [Thread-1884 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:42:09,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:42:09,636 DEBUG [Thread-1871 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:42:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:42:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:42:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:42:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:42:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:42:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:42:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:42:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:42:09,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:42:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:42:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:42:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:42:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:42:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:42:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:42:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:42:09,637 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:42:09,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:42:09,638 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37150, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:42:09,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:42:09,638 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:42:09,638 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:42:09,638 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37192, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:42:09,638 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:42:09,639 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:42:09,639 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:42:09,639 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:42:09,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:42:09,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:42:09,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:42:09,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:42:09,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:42:09,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:42:09,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:42:09,641 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:42:09,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:42:09,642 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:42:09,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:42:09,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:42:09,643 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2] 2024-11-23T15:42:09,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-23T15:42:09,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T15:42:09,644 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:42:09,644 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:42:09,645 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:42:09,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 2888821c5754525ea6530da454808a84 2024-11-23T15:42:09,648 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:09,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T15:42:09,668 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/a8b8895d80e445ec82d02bc18dff6e29 is 50, key is test_row_0/A:col10/1732376529646/Put/seqid=0 2024-11-23T15:42:09,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742238_1414 (size=12001) 2024-11-23T15:42:09,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376589684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:09,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:09,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376589684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:09,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376589685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:09,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376589686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:09,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376589687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:09,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T15:42:09,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-23T15:42:09,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:09,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:09,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:09,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:09,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:09,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:09,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376589794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:09,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:09,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376589805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:09,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:09,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376589805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:09,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:09,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376589805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:09,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:09,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376589805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:09,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:09,810 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:09,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-23T15:42:09,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:09,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:09,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:09,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:09,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T15:42:10,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:10,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376590004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:10,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:10,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376590014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:10,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376590014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:10,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376590014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:10,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376590014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:10,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:10,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/a8b8895d80e445ec82d02bc18dff6e29 2024-11-23T15:42:10,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/8c99019424494fafb34a01e6779d007a is 50, key is test_row_0/B:col10/1732376529646/Put/seqid=0 2024-11-23T15:42:10,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742239_1415 (size=12001) 2024-11-23T15:42:10,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-23T15:42:10,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:10,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:10,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:10,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:10,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:10,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:10,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-23T15:42:10,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:10,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:10,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:10,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:10,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:10,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T15:42:10,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:10,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376590314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:10,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:10,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376590323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376590324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:10,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376590324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376590324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:10,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-23T15:42:10,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:10,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:10,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:10,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:10,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:10,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/8c99019424494fafb34a01e6779d007a 2024-11-23T15:42:10,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/aa9e9463ad9c43c1bd507bc02091c717 is 50, key is test_row_0/C:col10/1732376529646/Put/seqid=0 2024-11-23T15:42:10,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742240_1416 (size=12001) 2024-11-23T15:42:10,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/aa9e9463ad9c43c1bd507bc02091c717 2024-11-23T15:42:10,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/a8b8895d80e445ec82d02bc18dff6e29 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/a8b8895d80e445ec82d02bc18dff6e29 2024-11-23T15:42:10,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/a8b8895d80e445ec82d02bc18dff6e29, entries=150, sequenceid=12, filesize=11.7 K 2024-11-23T15:42:10,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/8c99019424494fafb34a01e6779d007a as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8c99019424494fafb34a01e6779d007a 2024-11-23T15:42:10,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8c99019424494fafb34a01e6779d007a, entries=150, sequenceid=12, filesize=11.7 K 2024-11-23T15:42:10,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/aa9e9463ad9c43c1bd507bc02091c717 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/aa9e9463ad9c43c1bd507bc02091c717 2024-11-23T15:42:10,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/aa9e9463ad9c43c1bd507bc02091c717, entries=150, sequenceid=12, filesize=11.7 K 2024-11-23T15:42:10,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2888821c5754525ea6530da454808a84 in 890ms, sequenceid=12, compaction requested=false 2024-11-23T15:42:10,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:10,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-23T15:42:10,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:10,561 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T15:42:10,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/f527f4816ce240b1a8c56c5103490ff0 is 50, key is test_row_0/A:col10/1732376529683/Put/seqid=0 2024-11-23T15:42:10,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742241_1417 (size=12001) 2024-11-23T15:42:10,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T15:42:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 2888821c5754525ea6530da454808a84 2024-11-23T15:42:10,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:10,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376590839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:10,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:10,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376590840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376590841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376590842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:10,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376590843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,849 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:10,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376590954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:10,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376590954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376590954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:10,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376590955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:10,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:10,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:10,960 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376590955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:10,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:10,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:10,968 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/f527f4816ce240b1a8c56c5103490ff0 2024-11-23T15:42:10,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/5c78db8c7c9d4b58a8882fceb326c1f1 is 50, key is test_row_0/B:col10/1732376529683/Put/seqid=0 2024-11-23T15:42:10,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742242_1418 (size=12001) 2024-11-23T15:42:11,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376591164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:11,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:11,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376591165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:11,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:11,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:11,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:11,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376591165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:11,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:11,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:11,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376591165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:11,170 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:11,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:11,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:11,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:11,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376591165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:11,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:11,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:11,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:11,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:11,376 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/5c78db8c7c9d4b58a8882fceb326c1f1 2024-11-23T15:42:11,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/ece009d56f2444e79f098b9cddf8c9f0 is 50, key is test_row_0/C:col10/1732376529683/Put/seqid=0 2024-11-23T15:42:11,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742243_1419 (size=12001) 2024-11-23T15:42:11,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376591474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:11,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376591474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:11,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376591474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:11,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376591475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:11,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:11,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:11,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376591475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:11,479 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:11,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:11,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:11,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:11,786 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/ece009d56f2444e79f098b9cddf8c9f0 2024-11-23T15:42:11,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/f527f4816ce240b1a8c56c5103490ff0 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/f527f4816ce240b1a8c56c5103490ff0 2024-11-23T15:42:11,792 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/f527f4816ce240b1a8c56c5103490ff0, entries=150, sequenceid=37, filesize=11.7 K 2024-11-23T15:42:11,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/5c78db8c7c9d4b58a8882fceb326c1f1 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/5c78db8c7c9d4b58a8882fceb326c1f1 2024-11-23T15:42:11,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T15:42:11,796 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/5c78db8c7c9d4b58a8882fceb326c1f1, entries=150, sequenceid=37, filesize=11.7 K 2024-11-23T15:42:11,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/ece009d56f2444e79f098b9cddf8c9f0 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/ece009d56f2444e79f098b9cddf8c9f0 2024-11-23T15:42:11,799 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/ece009d56f2444e79f098b9cddf8c9f0, entries=150, sequenceid=37, filesize=11.7 K 2024-11-23T15:42:11,800 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2888821c5754525ea6530da454808a84 in 1239ms, sequenceid=37, compaction requested=false 2024-11-23T15:42:11,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:11,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:11,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-23T15:42:11,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-23T15:42:11,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-23T15:42:11,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1570 sec 2024-11-23T15:42:11,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.1670 sec 2024-11-23T15:42:11,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 2888821c5754525ea6530da454808a84 2024-11-23T15:42:11,989 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:11,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-23T15:42:11,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/93636576e19d4b2081d5e076a901d929 is 50, key is test_row_0/A:col10/1732376531988/Put/seqid=0 2024-11-23T15:42:11,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742244_1420 (size=12001) 2024-11-23T15:42:11,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/93636576e19d4b2081d5e076a901d929 2024-11-23T15:42:12,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/04188a02beff4f5ba45f7cf762c38e97 is 50, key is test_row_0/B:col10/1732376531988/Put/seqid=0 2024-11-23T15:42:12,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742245_1421 (size=12001) 2024-11-23T15:42:12,029 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T15:42:12,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376592025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:12,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:12,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376592025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:12,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376592026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:12,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376592027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:12,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376592032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:12,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:12,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376592144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:12,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376592144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376592144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376592145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:12,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:12,155 WARN [regionserver/b712f9af2c12:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-23T15:42:12,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:12,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376592154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:12,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:12,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376592354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:12,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376592364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376592364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:12,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:12,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376592364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:12,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:12,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:12,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376592364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:12,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:12,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/04188a02beff4f5ba45f7cf762c38e97 2024-11-23T15:42:12,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/e2694929361f49e7a7337297c61bc4f6 is 50, key is test_row_0/C:col10/1732376531988/Put/seqid=0 2024-11-23T15:42:12,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742246_1422 (size=12001) 2024-11-23T15:42:12,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376592664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:12,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376592674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376592675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:12,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376592684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:12,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376592684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:12,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:12,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/e2694929361f49e7a7337297c61bc4f6 2024-11-23T15:42:12,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/93636576e19d4b2081d5e076a901d929 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/93636576e19d4b2081d5e076a901d929 2024-11-23T15:42:12,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/93636576e19d4b2081d5e076a901d929, entries=150, sequenceid=50, filesize=11.7 K 2024-11-23T15:42:12,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/04188a02beff4f5ba45f7cf762c38e97 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/04188a02beff4f5ba45f7cf762c38e97 2024-11-23T15:42:12,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/04188a02beff4f5ba45f7cf762c38e97, entries=150, sequenceid=50, filesize=11.7 K 2024-11-23T15:42:12,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/e2694929361f49e7a7337297c61bc4f6 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/e2694929361f49e7a7337297c61bc4f6 2024-11-23T15:42:12,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/e2694929361f49e7a7337297c61bc4f6, entries=150, sequenceid=50, filesize=11.7 K 2024-11-23T15:42:12,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 2888821c5754525ea6530da454808a84 in 842ms, sequenceid=50, compaction requested=true 2024-11-23T15:42:12,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:12,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:42:12,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:12,831 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:12,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:42:12,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:12,831 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:12,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:42:12,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:12,832 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:12,832 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:12,832 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/A is initiating minor compaction (all files) 2024-11-23T15:42:12,832 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/B is initiating minor compaction (all files) 2024-11-23T15:42:12,832 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/A in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:12,832 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/B in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:12,832 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8c99019424494fafb34a01e6779d007a, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/5c78db8c7c9d4b58a8882fceb326c1f1, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/04188a02beff4f5ba45f7cf762c38e97] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=35.2 K 2024-11-23T15:42:12,832 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/a8b8895d80e445ec82d02bc18dff6e29, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/f527f4816ce240b1a8c56c5103490ff0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/93636576e19d4b2081d5e076a901d929] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=35.2 K 2024-11-23T15:42:12,832 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8b8895d80e445ec82d02bc18dff6e29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732376529646 2024-11-23T15:42:12,832 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c99019424494fafb34a01e6779d007a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732376529646 2024-11-23T15:42:12,833 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting f527f4816ce240b1a8c56c5103490ff0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732376529682 2024-11-23T15:42:12,833 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93636576e19d4b2081d5e076a901d929, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732376530839 2024-11-23T15:42:12,833 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c78db8c7c9d4b58a8882fceb326c1f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732376529682 2024-11-23T15:42:12,833 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 04188a02beff4f5ba45f7cf762c38e97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732376530839 2024-11-23T15:42:12,838 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#B#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:12,838 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/689fb86a242c4d219d7f9031630f211c is 50, key is test_row_0/B:col10/1732376531988/Put/seqid=0 2024-11-23T15:42:12,839 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#A#compaction#358 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:12,839 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/5bbd4e0e8bdc4f5e873c944a1b9e9239 is 50, key is test_row_0/A:col10/1732376531988/Put/seqid=0 2024-11-23T15:42:12,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742247_1423 (size=12104) 2024-11-23T15:42:12,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742248_1424 (size=12104) 2024-11-23T15:42:13,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 2888821c5754525ea6530da454808a84 2024-11-23T15:42:13,189 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:13,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T15:42:13,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/d2fd25cff7e84347939dade18569e4d9 is 50, key is test_row_0/A:col10/1732376532026/Put/seqid=0 2024-11-23T15:42:13,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742249_1425 (size=14341) 2024-11-23T15:42:13,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/d2fd25cff7e84347939dade18569e4d9 2024-11-23T15:42:13,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/5b583036c6e7457ca493205649473964 is 50, key is test_row_0/B:col10/1732376532026/Put/seqid=0 2024-11-23T15:42:13,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742250_1426 (size=12001) 2024-11-23T15:42:13,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:13,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376593209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:13,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376593209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:13,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376593209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:13,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:13,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376593210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:13,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:13,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:13,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376593213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:13,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:13,249 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/689fb86a242c4d219d7f9031630f211c as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/689fb86a242c4d219d7f9031630f211c 2024-11-23T15:42:13,254 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2888821c5754525ea6530da454808a84/B of 2888821c5754525ea6530da454808a84 into 689fb86a242c4d219d7f9031630f211c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:13,254 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:13,254 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/B, priority=13, startTime=1732376532831; duration=0sec 2024-11-23T15:42:13,254 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:13,254 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:B 2024-11-23T15:42:13,254 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:13,254 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/5bbd4e0e8bdc4f5e873c944a1b9e9239 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/5bbd4e0e8bdc4f5e873c944a1b9e9239 2024-11-23T15:42:13,255 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:13,255 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/C is initiating minor compaction (all files) 2024-11-23T15:42:13,255 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/C in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:13,255 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/aa9e9463ad9c43c1bd507bc02091c717, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/ece009d56f2444e79f098b9cddf8c9f0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/e2694929361f49e7a7337297c61bc4f6] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=35.2 K 2024-11-23T15:42:13,255 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting aa9e9463ad9c43c1bd507bc02091c717, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732376529646 2024-11-23T15:42:13,255 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting ece009d56f2444e79f098b9cddf8c9f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732376529682 2024-11-23T15:42:13,256 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting e2694929361f49e7a7337297c61bc4f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732376530839 2024-11-23T15:42:13,259 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2888821c5754525ea6530da454808a84/A of 2888821c5754525ea6530da454808a84 into 5bbd4e0e8bdc4f5e873c944a1b9e9239(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:13,259 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:13,259 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/A, priority=13, startTime=1732376532831; duration=0sec 2024-11-23T15:42:13,259 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:13,259 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:A 2024-11-23T15:42:13,264 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#C#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:13,264 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/3e17343bf7124cabba028128ad793412 is 50, key is test_row_0/C:col10/1732376531988/Put/seqid=0 2024-11-23T15:42:13,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742251_1427 (size=12104) 2024-11-23T15:42:13,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376593324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:13,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376593324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:13,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376593324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:13,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:13,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376593334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:13,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:13,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:13,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376593334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:13,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:13,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376593534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:13,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:13,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376593544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:13,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:13,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376593544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:13,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376593544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:13,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376593554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:13,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:13,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/5b583036c6e7457ca493205649473964 2024-11-23T15:42:13,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/b04be6f8f6b94c1fac571493d2df4b4e is 50, key is test_row_0/C:col10/1732376532026/Put/seqid=0 2024-11-23T15:42:13,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742252_1428 (size=12001) 2024-11-23T15:42:13,671 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/3e17343bf7124cabba028128ad793412 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/3e17343bf7124cabba028128ad793412 2024-11-23T15:42:13,675 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2888821c5754525ea6530da454808a84/C of 2888821c5754525ea6530da454808a84 into 3e17343bf7124cabba028128ad793412(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:13,675 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:13,675 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/C, priority=13, startTime=1732376532831; duration=0sec 2024-11-23T15:42:13,675 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:13,675 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:C 2024-11-23T15:42:13,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T15:42:13,815 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:13,815 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:42:13,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-23T15:42:13,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-23T15:42:13,817 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:42:13,817 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:42:13,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:42:13,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:13,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376593844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:13,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:13,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376593854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:13,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376593854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:13,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376593854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:13,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:13,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376593864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:13,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:13,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-23T15:42:13,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T15:42:13,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:13,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:13,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:13,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:13,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:13,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:14,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/b04be6f8f6b94c1fac571493d2df4b4e 2024-11-23T15:42:14,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/d2fd25cff7e84347939dade18569e4d9 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/d2fd25cff7e84347939dade18569e4d9 2024-11-23T15:42:14,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/d2fd25cff7e84347939dade18569e4d9, entries=200, sequenceid=74, filesize=14.0 K 2024-11-23T15:42:14,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/5b583036c6e7457ca493205649473964 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/5b583036c6e7457ca493205649473964 2024-11-23T15:42:14,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/5b583036c6e7457ca493205649473964, entries=150, sequenceid=74, filesize=11.7 K 2024-11-23T15:42:14,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/b04be6f8f6b94c1fac571493d2df4b4e as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b04be6f8f6b94c1fac571493d2df4b4e 2024-11-23T15:42:14,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b04be6f8f6b94c1fac571493d2df4b4e, entries=150, sequenceid=74, filesize=11.7 K 2024-11-23T15:42:14,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 2888821c5754525ea6530da454808a84 in 839ms, sequenceid=74, compaction requested=false 2024-11-23T15:42:14,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:14,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T15:42:14,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:14,122 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T15:42:14,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/7307937c4de14213a207e23df0836211 is 50, key is test_row_0/A:col10/1732376533189/Put/seqid=0 2024-11-23T15:42:14,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742253_1429 (size=12001) 2024-11-23T15:42:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-23T15:42:14,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 2888821c5754525ea6530da454808a84 2024-11-23T15:42:14,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:14,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376594399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:14,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376594400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376594400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:14,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:14,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:14,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376594403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:14,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,411 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:14,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:14,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376594405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,413 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-23T15:42:14,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376594514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:14,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376594514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:14,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376594514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:14,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376594514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:14,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:14,530 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/7307937c4de14213a207e23df0836211 2024-11-23T15:42:14,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376594524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:14,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/4d0eb928fbc643218ba4ebc419aee39e is 50, key is test_row_0/B:col10/1732376533189/Put/seqid=0 2024-11-23T15:42:14,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742254_1430 (size=12001) 2024-11-23T15:42:14,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376594724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376594724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376594724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,733 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:14,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376594734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:14,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376594734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:14,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:14,939 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/4d0eb928fbc643218ba4ebc419aee39e 2024-11-23T15:42:14,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/b7b5fde24323443080f83f5747236e19 is 50, key is test_row_0/C:col10/1732376533189/Put/seqid=0 2024-11-23T15:42:14,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742255_1431 (size=12001) 2024-11-23T15:42:14,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-23T15:42:15,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:15,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:15,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376595044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:15,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376595044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:15,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:15,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376595044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:15,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376595045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:15,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376595045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:15,348 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/b7b5fde24323443080f83f5747236e19 2024-11-23T15:42:15,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/7307937c4de14213a207e23df0836211 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/7307937c4de14213a207e23df0836211 2024-11-23T15:42:15,356 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/7307937c4de14213a207e23df0836211, entries=150, sequenceid=89, filesize=11.7 K 2024-11-23T15:42:15,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/4d0eb928fbc643218ba4ebc419aee39e as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/4d0eb928fbc643218ba4ebc419aee39e 2024-11-23T15:42:15,360 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/4d0eb928fbc643218ba4ebc419aee39e, entries=150, sequenceid=89, filesize=11.7 K 2024-11-23T15:42:15,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/b7b5fde24323443080f83f5747236e19 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b7b5fde24323443080f83f5747236e19 2024-11-23T15:42:15,365 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b7b5fde24323443080f83f5747236e19, entries=150, sequenceid=89, filesize=11.7 K 2024-11-23T15:42:15,365 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2888821c5754525ea6530da454808a84 in 1243ms, sequenceid=89, compaction requested=true 2024-11-23T15:42:15,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:15,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:15,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-23T15:42:15,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-23T15:42:15,368 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-23T15:42:15,368 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5500 sec 2024-11-23T15:42:15,370 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.5540 sec 2024-11-23T15:42:15,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 2888821c5754525ea6530da454808a84 2024-11-23T15:42:15,559 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:15,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T15:42:15,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/aedd0a7ec0ed48f788549abb637d2186 is 50, key is test_row_0/A:col10/1732376534403/Put/seqid=0 2024-11-23T15:42:15,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742256_1432 (size=14341) 2024-11-23T15:42:15,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:15,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376595570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376595571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:15,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376595572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:15,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:15,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376595581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:15,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376595582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:15,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:15,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,592 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,592 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:15,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376595684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:15,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376595684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:15,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376595684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:15,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:15,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376595694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:15,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:15,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376595694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:15,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:15,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:15,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376595894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:15,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376595894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:15,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376595894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:15,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:15,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376595904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:15,907 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:15,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:15,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376595904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:15,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:15,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:15,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-23T15:42:15,965 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:15,965 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:42:15,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-23T15:42:15,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/aedd0a7ec0ed48f788549abb637d2186 2024-11-23T15:42:15,967 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:42:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-23T15:42:15,967 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:42:15,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:42:15,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/150a99772b464d558ebe315d4db1f67c is 50, key is test_row_0/B:col10/1732376534403/Put/seqid=0 2024-11-23T15:42:15,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742257_1433 (size=12001) 2024-11-23T15:42:16,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-23T15:42:16,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-23T15:42:16,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:16,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:16,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:16,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:16,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:16,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:16,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:16,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376596204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:16,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:16,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:16,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:16,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376596204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:16,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:16,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376596204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:16,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:16,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:16,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376596214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:16,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:16,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376596214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:16,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:16,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:16,223 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:16,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-23T15:42:16,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:16,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:16,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:16,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:16,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:16,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-23T15:42:16,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/150a99772b464d558ebe315d4db1f67c 2024-11-23T15:42:16,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/52df32508cb04a748b7ca15cf1f6e343 is 50, key is test_row_0/C:col10/1732376534403/Put/seqid=0 2024-11-23T15:42:16,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742258_1434 (size=12001) 2024-11-23T15:42:16,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-23T15:42:16,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:16,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:16,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:16,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:16,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:16,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:16,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-23T15:42:16,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:16,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:16,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:16,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:16,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-23T15:42:16,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:16,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376596724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:16,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:16,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376596724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:16,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:16,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:16,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:16,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:16,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:16,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:16,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:16,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376596724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:16,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:16,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:16,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:16,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-23T15:42:16,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:16,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:16,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:16,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:16,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:16,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:16,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:16,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376596734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:16,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:16,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:16,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376596734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:16,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:16,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:16,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:16,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:16,739 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:16,786 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/52df32508cb04a748b7ca15cf1f6e343 2024-11-23T15:42:16,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/aedd0a7ec0ed48f788549abb637d2186 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/aedd0a7ec0ed48f788549abb637d2186 2024-11-23T15:42:16,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/aedd0a7ec0ed48f788549abb637d2186, entries=200, sequenceid=114, filesize=14.0 K 2024-11-23T15:42:16,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/150a99772b464d558ebe315d4db1f67c as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/150a99772b464d558ebe315d4db1f67c 2024-11-23T15:42:16,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/150a99772b464d558ebe315d4db1f67c, entries=150, sequenceid=114, filesize=11.7 K 2024-11-23T15:42:16,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/52df32508cb04a748b7ca15cf1f6e343 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/52df32508cb04a748b7ca15cf1f6e343 2024-11-23T15:42:16,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/52df32508cb04a748b7ca15cf1f6e343, entries=150, sequenceid=114, filesize=11.7 K 2024-11-23T15:42:16,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2888821c5754525ea6530da454808a84 in 1240ms, sequenceid=114, compaction requested=true 2024-11-23T15:42:16,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:16,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:42:16,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:16,800 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:42:16,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:42:16,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:16,800 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:42:16,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:42:16,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:16,801 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:42:16,801 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:42:16,801 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/B is initiating minor compaction (all files) 2024-11-23T15:42:16,801 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/A is initiating minor compaction (all files) 2024-11-23T15:42:16,801 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/B in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:16,801 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/A in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:16,801 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/689fb86a242c4d219d7f9031630f211c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/5b583036c6e7457ca493205649473964, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/4d0eb928fbc643218ba4ebc419aee39e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/150a99772b464d558ebe315d4db1f67c] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=47.0 K 2024-11-23T15:42:16,801 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/5bbd4e0e8bdc4f5e873c944a1b9e9239, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/d2fd25cff7e84347939dade18569e4d9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/7307937c4de14213a207e23df0836211, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/aedd0a7ec0ed48f788549abb637d2186] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=51.5 K 2024-11-23T15:42:16,801 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 689fb86a242c4d219d7f9031630f211c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732376530839 2024-11-23T15:42:16,801 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bbd4e0e8bdc4f5e873c944a1b9e9239, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732376530839 2024-11-23T15:42:16,801 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2fd25cff7e84347939dade18569e4d9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732376532012 2024-11-23T15:42:16,801 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b583036c6e7457ca493205649473964, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732376532026 2024-11-23T15:42:16,802 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d0eb928fbc643218ba4ebc419aee39e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732376533189 2024-11-23T15:42:16,802 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7307937c4de14213a207e23df0836211, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732376533189 2024-11-23T15:42:16,802 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting aedd0a7ec0ed48f788549abb637d2186, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732376534384 2024-11-23T15:42:16,802 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 150a99772b464d558ebe315d4db1f67c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732376534399 2024-11-23T15:42:16,809 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#B#compaction#369 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:16,810 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#A#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:16,810 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/fcf32fd8df75498eaf73c371a740823f is 50, key is test_row_0/A:col10/1732376534403/Put/seqid=0 2024-11-23T15:42:16,810 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/628453d35a6e442ebd3235308bb59592 is 50, key is test_row_0/B:col10/1732376534403/Put/seqid=0 2024-11-23T15:42:16,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742260_1436 (size=12241) 2024-11-23T15:42:16,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742259_1435 (size=12241) 2024-11-23T15:42:16,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-23T15:42:16,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:16,881 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T15:42:16,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/c31c8318944043dfb309dad19066f010 is 50, key is test_row_0/A:col10/1732376535580/Put/seqid=0 2024-11-23T15:42:16,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742261_1437 (size=12001) 2024-11-23T15:42:17,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-23T15:42:17,217 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/628453d35a6e442ebd3235308bb59592 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/628453d35a6e442ebd3235308bb59592 2024-11-23T15:42:17,223 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/fcf32fd8df75498eaf73c371a740823f as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/fcf32fd8df75498eaf73c371a740823f 2024-11-23T15:42:17,225 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2888821c5754525ea6530da454808a84/B of 2888821c5754525ea6530da454808a84 into 628453d35a6e442ebd3235308bb59592(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:17,225 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:17,225 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/B, priority=12, startTime=1732376536800; duration=0sec 2024-11-23T15:42:17,225 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:17,225 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:B 2024-11-23T15:42:17,225 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:42:17,226 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:42:17,226 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/C is initiating minor compaction (all files) 2024-11-23T15:42:17,226 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/C in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:17,226 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/3e17343bf7124cabba028128ad793412, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b04be6f8f6b94c1fac571493d2df4b4e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b7b5fde24323443080f83f5747236e19, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/52df32508cb04a748b7ca15cf1f6e343] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=47.0 K 2024-11-23T15:42:17,227 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e17343bf7124cabba028128ad793412, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732376530839 2024-11-23T15:42:17,227 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting b04be6f8f6b94c1fac571493d2df4b4e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732376532026 2024-11-23T15:42:17,227 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2888821c5754525ea6530da454808a84/A of 2888821c5754525ea6530da454808a84 into fcf32fd8df75498eaf73c371a740823f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:17,227 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:17,227 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/A, priority=12, startTime=1732376536800; duration=0sec 2024-11-23T15:42:17,227 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:17,227 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:A 2024-11-23T15:42:17,227 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting b7b5fde24323443080f83f5747236e19, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732376533189 2024-11-23T15:42:17,228 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 52df32508cb04a748b7ca15cf1f6e343, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732376534399 2024-11-23T15:42:17,234 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#C#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:17,235 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/8832ef1a867a487db41d560089cc4978 is 50, key is test_row_0/C:col10/1732376534403/Put/seqid=0 2024-11-23T15:42:17,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742262_1438 (size=12241) 2024-11-23T15:42:17,290 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/c31c8318944043dfb309dad19066f010 2024-11-23T15:42:17,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/2f50c842eae84416abfe5a2f275edb31 is 50, key is test_row_0/B:col10/1732376535580/Put/seqid=0 2024-11-23T15:42:17,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742263_1439 (size=12001) 2024-11-23T15:42:17,642 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/8832ef1a867a487db41d560089cc4978 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/8832ef1a867a487db41d560089cc4978 2024-11-23T15:42:17,647 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2888821c5754525ea6530da454808a84/C of 2888821c5754525ea6530da454808a84 into 8832ef1a867a487db41d560089cc4978(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:17,647 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:17,647 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/C, priority=12, startTime=1732376536800; duration=0sec 2024-11-23T15:42:17,647 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:17,647 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:C 2024-11-23T15:42:17,698 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/2f50c842eae84416abfe5a2f275edb31 2024-11-23T15:42:17,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/b66a1c8d17344a47938c8d0d3983cb4a is 50, key is test_row_0/C:col10/1732376535580/Put/seqid=0 2024-11-23T15:42:17,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742264_1440 (size=12001) 2024-11-23T15:42:17,707 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/b66a1c8d17344a47938c8d0d3983cb4a 2024-11-23T15:42:17,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/c31c8318944043dfb309dad19066f010 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/c31c8318944043dfb309dad19066f010 2024-11-23T15:42:17,714 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/c31c8318944043dfb309dad19066f010, entries=150, sequenceid=125, filesize=11.7 K 2024-11-23T15:42:17,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/2f50c842eae84416abfe5a2f275edb31 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/2f50c842eae84416abfe5a2f275edb31 2024-11-23T15:42:17,724 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/2f50c842eae84416abfe5a2f275edb31, entries=150, sequenceid=125, filesize=11.7 K 2024-11-23T15:42:17,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/b66a1c8d17344a47938c8d0d3983cb4a as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b66a1c8d17344a47938c8d0d3983cb4a 2024-11-23T15:42:17,730 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b66a1c8d17344a47938c8d0d3983cb4a, entries=150, sequenceid=125, filesize=11.7 K 2024-11-23T15:42:17,731 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 2888821c5754525ea6530da454808a84 in 850ms, sequenceid=125, compaction requested=false 2024-11-23T15:42:17,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:17,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:17,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-23T15:42:17,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-23T15:42:17,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-23T15:42:17,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7650 sec 2024-11-23T15:42:17,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.7690 sec 2024-11-23T15:42:17,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 2888821c5754525ea6530da454808a84 2024-11-23T15:42:17,745 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:17,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T15:42:17,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/0754e376450340f8abf0adef5838ae81 is 50, key is test_row_0/A:col10/1732376537744/Put/seqid=0 2024-11-23T15:42:17,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742265_1441 (size=14541) 2024-11-23T15:42:17,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376597814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:17,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376597815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:17,829 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:17,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376597828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:17,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:17,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376597828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:17,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376597829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:17,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:17,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376597934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:17,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376597934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:17,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:17,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376597934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:17,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:17,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376597945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:17,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:17,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:17,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:17,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376597945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:18,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-23T15:42:18,114 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:18,115 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:42:18,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-23T15:42:18,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-23T15:42:18,116 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:42:18,117 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:42:18,117 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:42:18,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376598144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:18,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/0754e376450340f8abf0adef5838ae81 2024-11-23T15:42:18,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:18,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/24d4dc652d234205a85d1f6415b911a4 is 50, key is test_row_0/B:col10/1732376537744/Put/seqid=0 2024-11-23T15:42:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376598154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:18,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:18,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376598154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:18,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:18,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,160 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:18,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742266_1442 (size=12151) 2024-11-23T15:42:18,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376598164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:18,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:18,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,169 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376598164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:18,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:18,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-23T15:42:18,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-23T15:42:18,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:18,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:18,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:18,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:18,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:18,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-23T15:42:18,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:18,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:18,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:18,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:18,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:18,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:18,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-23T15:42:18,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:18,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376598454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:18,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:18,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:18,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376598464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:18,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:18,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,470 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:18,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376598464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:18,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,470 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:18,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376598474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:18,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:18,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376598474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:18,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:18,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:18,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/24d4dc652d234205a85d1f6415b911a4 2024-11-23T15:42:18,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/7ce25bf8a9d2457fa2ff4ee9cc191c35 is 50, key is test_row_0/C:col10/1732376537744/Put/seqid=0 2024-11-23T15:42:18,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742267_1443 (size=12151) 2024-11-23T15:42:18,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-23T15:42:18,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:18,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:18,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:18,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:18,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:18,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-23T15:42:18,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:18,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:18,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:18,728 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:18,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-23T15:42:18,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-23T15:42:18,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:18,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:18,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:18,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:18,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:18,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/7ce25bf8a9d2457fa2ff4ee9cc191c35 2024-11-23T15:42:18,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/0754e376450340f8abf0adef5838ae81 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/0754e376450340f8abf0adef5838ae81 2024-11-23T15:42:18,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/0754e376450340f8abf0adef5838ae81, entries=200, sequenceid=139, filesize=14.2 K 2024-11-23T15:42:18,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/24d4dc652d234205a85d1f6415b911a4 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/24d4dc652d234205a85d1f6415b911a4 2024-11-23T15:42:18,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/24d4dc652d234205a85d1f6415b911a4, entries=150, sequenceid=139, filesize=11.9 K 2024-11-23T15:42:18,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/7ce25bf8a9d2457fa2ff4ee9cc191c35 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/7ce25bf8a9d2457fa2ff4ee9cc191c35 2024-11-23T15:42:18,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376598974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:18,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376598974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:18,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376598974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:18,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:18,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/7ce25bf8a9d2457fa2ff4ee9cc191c35, entries=150, sequenceid=139, filesize=11.9 K 2024-11-23T15:42:18,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 2888821c5754525ea6530da454808a84 in 1245ms, sequenceid=139, compaction requested=true 2024-11-23T15:42:18,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:18,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:42:18,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:18,991 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:18,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:42:18,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:18,991 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:18,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:42:18,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:18,991 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:18,991 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:18,991 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/A is initiating minor compaction (all files) 2024-11-23T15:42:18,991 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/B is initiating minor compaction (all files) 2024-11-23T15:42:18,991 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/B in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:18,991 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/A in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:18,992 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/fcf32fd8df75498eaf73c371a740823f, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/c31c8318944043dfb309dad19066f010, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/0754e376450340f8abf0adef5838ae81] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=37.9 K 2024-11-23T15:42:18,992 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/628453d35a6e442ebd3235308bb59592, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/2f50c842eae84416abfe5a2f275edb31, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/24d4dc652d234205a85d1f6415b911a4] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=35.5 K 2024-11-23T15:42:18,992 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcf32fd8df75498eaf73c371a740823f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732376534399 2024-11-23T15:42:18,992 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 628453d35a6e442ebd3235308bb59592, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732376534399 2024-11-23T15:42:18,992 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f50c842eae84416abfe5a2f275edb31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732376535571 2024-11-23T15:42:18,992 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting c31c8318944043dfb309dad19066f010, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732376535571 2024-11-23T15:42:18,992 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 24d4dc652d234205a85d1f6415b911a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732376537742 2024-11-23T15:42:18,992 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0754e376450340f8abf0adef5838ae81, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732376537742 2024-11-23T15:42:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 2888821c5754525ea6530da454808a84 2024-11-23T15:42:18,993 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:18,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-23T15:42:18,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/310bde306a98481f9432296cb9e82a40 is 50, key is test_row_0/A:col10/1732376538993/Put/seqid=0 2024-11-23T15:42:19,000 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#B#compaction#379 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:19,000 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#A#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:19,000 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/eba2506262d34dd3b258c9357c4e6503 is 50, key is test_row_0/B:col10/1732376537744/Put/seqid=0 2024-11-23T15:42:19,000 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/dd2d67c5f0d54e90aea809116f04f16b is 50, key is test_row_0/A:col10/1732376537744/Put/seqid=0 2024-11-23T15:42:19,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742269_1445 (size=12493) 2024-11-23T15:42:19,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742268_1444 (size=16931) 2024-11-23T15:42:19,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742270_1446 (size=12493) 2024-11-23T15:42:19,008 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/eba2506262d34dd3b258c9357c4e6503 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/eba2506262d34dd3b258c9357c4e6503 2024-11-23T15:42:19,011 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2888821c5754525ea6530da454808a84/B of 2888821c5754525ea6530da454808a84 into eba2506262d34dd3b258c9357c4e6503(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:19,011 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:19,012 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/B, priority=13, startTime=1732376538991; duration=0sec 2024-11-23T15:42:19,012 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:19,012 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:B 2024-11-23T15:42:19,012 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:19,013 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:19,013 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/C is initiating minor compaction (all files) 2024-11-23T15:42:19,014 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/C in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:19,014 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/8832ef1a867a487db41d560089cc4978, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b66a1c8d17344a47938c8d0d3983cb4a, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/7ce25bf8a9d2457fa2ff4ee9cc191c35] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=35.5 K 2024-11-23T15:42:19,014 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8832ef1a867a487db41d560089cc4978, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732376534399 2024-11-23T15:42:19,014 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting b66a1c8d17344a47938c8d0d3983cb4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732376535571 2024-11-23T15:42:19,014 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ce25bf8a9d2457fa2ff4ee9cc191c35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732376537742 2024-11-23T15:42:19,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376599007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:19,019 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#C#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:19,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:19,019 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/823f8a67536c4083bca2717f341d595a is 50, key is test_row_0/C:col10/1732376537744/Put/seqid=0 2024-11-23T15:42:19,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376599013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:19,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:19,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742271_1447 (size=12493) 2024-11-23T15:42:19,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-23T15:42:19,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:19,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:19,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:19,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:19,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:19,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376599124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:19,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:19,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376599124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:19,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:19,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:19,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-23T15:42:19,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:19,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:19,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:19,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:19,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:19,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-23T15:42:19,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376599334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:19,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376599334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:19,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-23T15:42:19,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:19,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:19,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:19,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:19,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:19,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/310bde306a98481f9432296cb9e82a40 2024-11-23T15:42:19,408 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/dd2d67c5f0d54e90aea809116f04f16b as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/dd2d67c5f0d54e90aea809116f04f16b 2024-11-23T15:42:19,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/e6db866b55a642e5ab3f39c5d0fe81e9 is 50, key is test_row_0/B:col10/1732376538993/Put/seqid=0 2024-11-23T15:42:19,412 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2888821c5754525ea6530da454808a84/A of 2888821c5754525ea6530da454808a84 into dd2d67c5f0d54e90aea809116f04f16b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:19,412 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:19,412 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/A, priority=13, startTime=1732376538991; duration=0sec 2024-11-23T15:42:19,412 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:19,413 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:A 2024-11-23T15:42:19,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742272_1448 (size=12151) 2024-11-23T15:42:19,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/e6db866b55a642e5ab3f39c5d0fe81e9 2024-11-23T15:42:19,426 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/823f8a67536c4083bca2717f341d595a as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/823f8a67536c4083bca2717f341d595a 2024-11-23T15:42:19,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/0a90da16991148c8b6d977c173276869 is 50, key is test_row_0/C:col10/1732376538993/Put/seqid=0 2024-11-23T15:42:19,430 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2888821c5754525ea6530da454808a84/C of 2888821c5754525ea6530da454808a84 into 823f8a67536c4083bca2717f341d595a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:19,430 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:19,430 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/C, priority=13, startTime=1732376538991; duration=0sec 2024-11-23T15:42:19,430 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:19,430 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:C 2024-11-23T15:42:19,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742273_1449 (size=12151) 2024-11-23T15:42:19,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/0a90da16991148c8b6d977c173276869 2024-11-23T15:42:19,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/310bde306a98481f9432296cb9e82a40 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/310bde306a98481f9432296cb9e82a40 2024-11-23T15:42:19,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/310bde306a98481f9432296cb9e82a40, entries=250, sequenceid=167, filesize=16.5 K 2024-11-23T15:42:19,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/e6db866b55a642e5ab3f39c5d0fe81e9 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/e6db866b55a642e5ab3f39c5d0fe81e9 2024-11-23T15:42:19,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/e6db866b55a642e5ab3f39c5d0fe81e9, entries=150, sequenceid=167, filesize=11.9 K 2024-11-23T15:42:19,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/0a90da16991148c8b6d977c173276869 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/0a90da16991148c8b6d977c173276869 2024-11-23T15:42:19,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/0a90da16991148c8b6d977c173276869, entries=150, sequenceid=167, filesize=11.9 K 2024-11-23T15:42:19,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 2888821c5754525ea6530da454808a84 in 463ms, sequenceid=167, compaction requested=false 2024-11-23T15:42:19,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:19,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-23T15:42:19,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:19,490 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-23T15:42:19,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/2db7138e86304d13959a7ba64622b225 is 50, key is test_row_0/A:col10/1732376539005/Put/seqid=0 2024-11-23T15:42:19,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742274_1450 (size=12151) 2024-11-23T15:42:19,501 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/2db7138e86304d13959a7ba64622b225 2024-11-23T15:42:19,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/0d77402f495747b6b218679067555b27 is 50, key is test_row_0/B:col10/1732376539005/Put/seqid=0 2024-11-23T15:42:19,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742275_1451 (size=12151) 2024-11-23T15:42:19,514 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/0d77402f495747b6b218679067555b27 2024-11-23T15:42:19,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/2ce6a9f541014b7dbe6256c618a76b8a is 50, key is test_row_0/C:col10/1732376539005/Put/seqid=0 2024-11-23T15:42:19,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742276_1452 (size=12151) 2024-11-23T15:42:19,526 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/2ce6a9f541014b7dbe6256c618a76b8a 2024-11-23T15:42:19,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/2db7138e86304d13959a7ba64622b225 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/2db7138e86304d13959a7ba64622b225 2024-11-23T15:42:19,534 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/2db7138e86304d13959a7ba64622b225, entries=150, sequenceid=178, filesize=11.9 K 2024-11-23T15:42:19,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/0d77402f495747b6b218679067555b27 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/0d77402f495747b6b218679067555b27 2024-11-23T15:42:19,538 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/0d77402f495747b6b218679067555b27, entries=150, sequenceid=178, filesize=11.9 K 2024-11-23T15:42:19,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/2ce6a9f541014b7dbe6256c618a76b8a as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/2ce6a9f541014b7dbe6256c618a76b8a 2024-11-23T15:42:19,541 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/2ce6a9f541014b7dbe6256c618a76b8a, entries=150, sequenceid=178, filesize=11.9 K 2024-11-23T15:42:19,542 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 2888821c5754525ea6530da454808a84 in 52ms, sequenceid=178, compaction requested=true 2024-11-23T15:42:19,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:19,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:19,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-23T15:42:19,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-23T15:42:19,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-23T15:42:19,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4260 sec 2024-11-23T15:42:19,549 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.4320 sec 2024-11-23T15:42:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 2888821c5754525ea6530da454808a84 2024-11-23T15:42:19,661 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:19,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T15:42:19,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/5702fde3f72542c5b747a5ade3cd016c is 50, key is test_row_0/A:col10/1732376539660/Put/seqid=0 2024-11-23T15:42:19,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742277_1453 (size=16931) 2024-11-23T15:42:19,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:19,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376599759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:19,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:19,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:19,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376599759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:19,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:19,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:19,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376599874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:19,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:19,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376599874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:19,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:19,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:19,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:19,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376599994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:19,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:19,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376599994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:19,998 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_0' in 2888821c5754525ea6530da454808a84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:42:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:19,998 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_2' in 2888821c5754525ea6530da454808a84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:42:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:20,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376600004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:20,009 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_0' in 2888821c5754525ea6530da454808a84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:42:20,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:20,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:20,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:20,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/5702fde3f72542c5b747a5ade3cd016c 2024-11-23T15:42:20,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/d77c23e4b0034c2e80e6dfb11c4c2947 is 50, key is test_row_0/B:col10/1732376539660/Put/seqid=0 2024-11-23T15:42:20,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742278_1454 (size=12151) 2024-11-23T15:42:20,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:20,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376600084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:20,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:20,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376600094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:20,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-23T15:42:20,264 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:20,264 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:42:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-23T15:42:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-23T15:42:20,266 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:42:20,267 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:42:20,267 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:42:20,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-23T15:42:20,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:20,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376600404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:20,405 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:20,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:20,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-23T15:42:20,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:20,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:20,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:20,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:20,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:20,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:20,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376600414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:20,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:20,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:20,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:20,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/d77c23e4b0034c2e80e6dfb11c4c2947 2024-11-23T15:42:20,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/b529406f53e443f7a70a3bc5b3650ec6 is 50, key is test_row_0/C:col10/1732376539660/Put/seqid=0 2024-11-23T15:42:20,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742279_1455 (size=12151) 2024-11-23T15:42:20,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-23T15:42:20,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:20,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:20,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:20,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:20,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:20,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-23T15:42:20,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-23T15:42:20,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:20,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:20,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:20,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:20,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:20,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-23T15:42:20,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:20,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:20,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:20,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:20,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:20,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/b529406f53e443f7a70a3bc5b3650ec6 2024-11-23T15:42:20,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/5702fde3f72542c5b747a5ade3cd016c as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/5702fde3f72542c5b747a5ade3cd016c 2024-11-23T15:42:20,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/5702fde3f72542c5b747a5ade3cd016c, entries=250, sequenceid=189, filesize=16.5 K 2024-11-23T15:42:20,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/d77c23e4b0034c2e80e6dfb11c4c2947 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/d77c23e4b0034c2e80e6dfb11c4c2947 2024-11-23T15:42:20,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-23T15:42:20,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/d77c23e4b0034c2e80e6dfb11c4c2947, entries=150, sequenceid=189, filesize=11.9 K 2024-11-23T15:42:20,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/b529406f53e443f7a70a3bc5b3650ec6 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b529406f53e443f7a70a3bc5b3650ec6 2024-11-23T15:42:20,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b529406f53e443f7a70a3bc5b3650ec6, entries=150, sequenceid=189, filesize=11.9 K 2024-11-23T15:42:20,901 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2888821c5754525ea6530da454808a84 in 1240ms, sequenceid=189, compaction requested=true 2024-11-23T15:42:20,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:20,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:42:20,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:20,901 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:42:20,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:42:20,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:20,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:42:20,901 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:42:20,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:20,902 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 58506 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:42:20,903 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/A is initiating minor compaction (all files) 2024-11-23T15:42:20,903 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:42:20,903 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/A in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:20,903 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/B is initiating minor compaction (all files) 2024-11-23T15:42:20,903 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/B in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:20,903 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/dd2d67c5f0d54e90aea809116f04f16b, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/310bde306a98481f9432296cb9e82a40, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/2db7138e86304d13959a7ba64622b225, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/5702fde3f72542c5b747a5ade3cd016c] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=57.1 K 2024-11-23T15:42:20,903 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/eba2506262d34dd3b258c9357c4e6503, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/e6db866b55a642e5ab3f39c5d0fe81e9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/0d77402f495747b6b218679067555b27, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/d77c23e4b0034c2e80e6dfb11c4c2947] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=47.8 K 2024-11-23T15:42:20,903 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd2d67c5f0d54e90aea809116f04f16b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732376537742 2024-11-23T15:42:20,903 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting eba2506262d34dd3b258c9357c4e6503, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732376537742 2024-11-23T15:42:20,903 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 310bde306a98481f9432296cb9e82a40, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732376537814 2024-11-23T15:42:20,903 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting e6db866b55a642e5ab3f39c5d0fe81e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732376537814 2024-11-23T15:42:20,904 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2db7138e86304d13959a7ba64622b225, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732376538997 2024-11-23T15:42:20,904 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d77402f495747b6b218679067555b27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732376538997 2024-11-23T15:42:20,904 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5702fde3f72542c5b747a5ade3cd016c, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732376539646 2024-11-23T15:42:20,904 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting d77c23e4b0034c2e80e6dfb11c4c2947, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732376539652 2024-11-23T15:42:20,911 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#B#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:20,911 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#A#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:20,912 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/8a2e18662e5b4f7fa229cb3ff51576d4 is 50, key is test_row_0/B:col10/1732376539660/Put/seqid=0 2024-11-23T15:42:20,912 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/78a8dc2870a74412964c91bbb8008210 is 50, key is test_row_0/A:col10/1732376539660/Put/seqid=0 2024-11-23T15:42:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 2888821c5754525ea6530da454808a84 2024-11-23T15:42:20,920 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:20,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T15:42:20,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742280_1456 (size=12629) 2024-11-23T15:42:20,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/670fe19dfabb4d1fb7773259be9acf61 is 50, key is test_row_0/A:col10/1732376539758/Put/seqid=0 2024-11-23T15:42:20,929 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/8a2e18662e5b4f7fa229cb3ff51576d4 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8a2e18662e5b4f7fa229cb3ff51576d4 2024-11-23T15:42:20,933 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2888821c5754525ea6530da454808a84/B of 2888821c5754525ea6530da454808a84 into 8a2e18662e5b4f7fa229cb3ff51576d4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:20,933 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:20,933 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/B, priority=12, startTime=1732376540901; duration=0sec 2024-11-23T15:42:20,933 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:20,933 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:B 2024-11-23T15:42:20,934 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:42:20,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742282_1458 (size=14541) 2024-11-23T15:42:20,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/670fe19dfabb4d1fb7773259be9acf61 2024-11-23T15:42:20,936 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:42:20,936 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/C is initiating minor compaction (all files) 2024-11-23T15:42:20,936 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/C in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:20,936 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/823f8a67536c4083bca2717f341d595a, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/0a90da16991148c8b6d977c173276869, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/2ce6a9f541014b7dbe6256c618a76b8a, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b529406f53e443f7a70a3bc5b3650ec6] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=47.8 K 2024-11-23T15:42:20,936 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 823f8a67536c4083bca2717f341d595a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732376537742 2024-11-23T15:42:20,936 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a90da16991148c8b6d977c173276869, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732376537814 2024-11-23T15:42:20,936 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ce6a9f541014b7dbe6256c618a76b8a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1732376538997 2024-11-23T15:42:20,937 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting b529406f53e443f7a70a3bc5b3650ec6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732376539652 2024-11-23T15:42:20,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742281_1457 (size=12629) 2024-11-23T15:42:20,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/b1c255e554c547cebc23fc0c92849ea1 is 50, key is test_row_0/B:col10/1732376539758/Put/seqid=0 2024-11-23T15:42:20,943 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/78a8dc2870a74412964c91bbb8008210 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/78a8dc2870a74412964c91bbb8008210 2024-11-23T15:42:20,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742283_1459 (size=12151) 2024-11-23T15:42:20,948 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#C#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:20,948 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/9f2012a7307f4d98ba8a823c2562688e is 50, key is test_row_0/C:col10/1732376539660/Put/seqid=0 2024-11-23T15:42:20,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/b1c255e554c547cebc23fc0c92849ea1 2024-11-23T15:42:20,950 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2888821c5754525ea6530da454808a84/A of 2888821c5754525ea6530da454808a84 into 78a8dc2870a74412964c91bbb8008210(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:20,950 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:20,951 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/A, priority=12, startTime=1732376540901; duration=0sec 2024-11-23T15:42:20,951 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:20,951 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:A 2024-11-23T15:42:20,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742284_1460 (size=12629) 2024-11-23T15:42:20,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/63d04a29356949b8b7e963467b929db9 is 50, key is test_row_0/C:col10/1732376539758/Put/seqid=0 2024-11-23T15:42:20,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742285_1461 (size=12151) 2024-11-23T15:42:20,961 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/9f2012a7307f4d98ba8a823c2562688e as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/9f2012a7307f4d98ba8a823c2562688e 2024-11-23T15:42:20,964 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2888821c5754525ea6530da454808a84/C of 2888821c5754525ea6530da454808a84 into 9f2012a7307f4d98ba8a823c2562688e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:20,964 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:20,964 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/C, priority=12, startTime=1732376540901; duration=0sec 2024-11-23T15:42:20,964 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:20,964 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:C 2024-11-23T15:42:20,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376600983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:20,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376600983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:20,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:20,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:20,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:20,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:20,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:20,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:21,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-23T15:42:21,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:21,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:21,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:21,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:21,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:21,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:21,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:21,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376601094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:21,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:21,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376601094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:21,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-23T15:42:21,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:21,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:21,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:21,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:21,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:21,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:21,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376601304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:21,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:21,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376601304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:21,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:21,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-23T15:42:21,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:21,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:21,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:21,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:21,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:21,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:21,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/63d04a29356949b8b7e963467b929db9 2024-11-23T15:42:21,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/670fe19dfabb4d1fb7773259be9acf61 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/670fe19dfabb4d1fb7773259be9acf61 2024-11-23T15:42:21,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/670fe19dfabb4d1fb7773259be9acf61, entries=200, sequenceid=215, filesize=14.2 K 2024-11-23T15:42:21,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/b1c255e554c547cebc23fc0c92849ea1 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/b1c255e554c547cebc23fc0c92849ea1 2024-11-23T15:42:21,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/b1c255e554c547cebc23fc0c92849ea1, entries=150, sequenceid=215, filesize=11.9 K 2024-11-23T15:42:21,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/63d04a29356949b8b7e963467b929db9 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/63d04a29356949b8b7e963467b929db9 2024-11-23T15:42:21,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/63d04a29356949b8b7e963467b929db9, entries=150, sequenceid=215, filesize=11.9 K 2024-11-23T15:42:21,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 2888821c5754525ea6530da454808a84 in 455ms, sequenceid=215, compaction requested=false 2024-11-23T15:42:21,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-23T15:42:21,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-23T15:42:21,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:21,488 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T15:42:21,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/9d8b25e6ebfe42fdb7812c1365f0ca7c is 50, key is test_row_0/A:col10/1732376540959/Put/seqid=0 2024-11-23T15:42:21,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742286_1462 (size=12151) 2024-11-23T15:42:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 2888821c5754525ea6530da454808a84 2024-11-23T15:42:21,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:21,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376601697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:21,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376601698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:21,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:21,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376601814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:21,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376601814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:21,820 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:21,895 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/9d8b25e6ebfe42fdb7812c1365f0ca7c 2024-11-23T15:42:21,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/8c36375129ce4be6a02c1bda02a9ef69 is 50, key is test_row_0/B:col10/1732376540959/Put/seqid=0 2024-11-23T15:42:21,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742287_1463 (size=12151) 2024-11-23T15:42:22,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376602014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:22,019 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_0' in 2888821c5754525ea6530da454808a84 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:42:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:22,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376602024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:22,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376602024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:22,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376602024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:22,031 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_2' in 2888821c5754525ea6530da454808a84 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:42:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:22,032 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_0' in 2888821c5754525ea6530da454808a84 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:42:22,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376602024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:22,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:22,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:22,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:22,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:22,304 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/8c36375129ce4be6a02c1bda02a9ef69 2024-11-23T15:42:22,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/2e7a08099b604e4ab9e1df8b4b83b752 is 50, key is test_row_0/C:col10/1732376540959/Put/seqid=0 2024-11-23T15:42:22,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742288_1464 (size=12151) 2024-11-23T15:42:22,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376602334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:22,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376602344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:22,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:22,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:22,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-23T15:42:22,712 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/2e7a08099b604e4ab9e1df8b4b83b752 2024-11-23T15:42:22,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/9d8b25e6ebfe42fdb7812c1365f0ca7c as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/9d8b25e6ebfe42fdb7812c1365f0ca7c 2024-11-23T15:42:22,719 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/9d8b25e6ebfe42fdb7812c1365f0ca7c, entries=150, sequenceid=229, filesize=11.9 K 2024-11-23T15:42:22,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/8c36375129ce4be6a02c1bda02a9ef69 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8c36375129ce4be6a02c1bda02a9ef69 2024-11-23T15:42:22,722 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8c36375129ce4be6a02c1bda02a9ef69, entries=150, sequenceid=229, filesize=11.9 K 2024-11-23T15:42:22,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/2e7a08099b604e4ab9e1df8b4b83b752 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/2e7a08099b604e4ab9e1df8b4b83b752 2024-11-23T15:42:22,726 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/2e7a08099b604e4ab9e1df8b4b83b752, entries=150, sequenceid=229, filesize=11.9 K 2024-11-23T15:42:22,726 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2888821c5754525ea6530da454808a84 in 1238ms, sequenceid=229, compaction requested=true 2024-11-23T15:42:22,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:22,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:22,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-23T15:42:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-23T15:42:22,729 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-23T15:42:22,729 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4610 sec 2024-11-23T15:42:22,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.4650 sec 2024-11-23T15:42:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 2888821c5754525ea6530da454808a84 2024-11-23T15:42:22,849 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:22,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T15:42:22,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/1df7158b04274d1785eaf46c4d7eb735 is 50, key is test_row_0/A:col10/1732376541690/Put/seqid=0 2024-11-23T15:42:22,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742289_1465 (size=14541) 2024-11-23T15:42:22,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376602874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:22,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:22,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376602878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:22,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:22,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376602984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:22,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376602994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:22,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:23,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376603194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:23,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:23,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:23,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:23,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376603204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:23,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:23,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:23,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:23,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/1df7158b04274d1785eaf46c4d7eb735 2024-11-23T15:42:23,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/2112ab8e48aa433cb3ff807f6b833369 is 50, key is test_row_0/B:col10/1732376541690/Put/seqid=0 2024-11-23T15:42:23,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742290_1466 (size=12151) 2024-11-23T15:42:23,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376603514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:23,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376603514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:23,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:23,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:23,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:23,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:23,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:23,519 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:23,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/2112ab8e48aa433cb3ff807f6b833369 2024-11-23T15:42:23,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/fed8bfc8d3c342b7bf588dc596c77ced is 50, key is test_row_0/C:col10/1732376541690/Put/seqid=0 2024-11-23T15:42:23,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742291_1467 (size=12151) 2024-11-23T15:42:24,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376604024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:24,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:24,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:24,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:24,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:24,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376604034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:24,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:24,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:24,076 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/fed8bfc8d3c342b7bf588dc596c77ced 2024-11-23T15:42:24,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/1df7158b04274d1785eaf46c4d7eb735 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/1df7158b04274d1785eaf46c4d7eb735 2024-11-23T15:42:24,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/1df7158b04274d1785eaf46c4d7eb735, entries=200, sequenceid=255, filesize=14.2 K 2024-11-23T15:42:24,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/2112ab8e48aa433cb3ff807f6b833369 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/2112ab8e48aa433cb3ff807f6b833369 2024-11-23T15:42:24,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/2112ab8e48aa433cb3ff807f6b833369, entries=150, sequenceid=255, filesize=11.9 K 2024-11-23T15:42:24,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/fed8bfc8d3c342b7bf588dc596c77ced as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/fed8bfc8d3c342b7bf588dc596c77ced 2024-11-23T15:42:24,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/fed8bfc8d3c342b7bf588dc596c77ced, entries=150, sequenceid=255, filesize=11.9 K 2024-11-23T15:42:24,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 2888821c5754525ea6530da454808a84 in 1242ms, sequenceid=255, compaction requested=true 2024-11-23T15:42:24,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:24,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:42:24,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:24,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:42:24,091 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:42:24,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:24,091 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:42:24,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:42:24,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:24,092 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53862 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:42:24,092 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49082 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:42:24,092 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/B is initiating minor compaction (all files) 2024-11-23T15:42:24,092 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/A is initiating minor compaction (all files) 2024-11-23T15:42:24,092 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/A in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:24,092 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/B in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:24,092 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8a2e18662e5b4f7fa229cb3ff51576d4, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/b1c255e554c547cebc23fc0c92849ea1, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8c36375129ce4be6a02c1bda02a9ef69, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/2112ab8e48aa433cb3ff807f6b833369] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=47.9 K 2024-11-23T15:42:24,092 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/78a8dc2870a74412964c91bbb8008210, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/670fe19dfabb4d1fb7773259be9acf61, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/9d8b25e6ebfe42fdb7812c1365f0ca7c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/1df7158b04274d1785eaf46c4d7eb735] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=52.6 K 2024-11-23T15:42:24,093 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78a8dc2870a74412964c91bbb8008210, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732376539652 2024-11-23T15:42:24,093 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a2e18662e5b4f7fa229cb3ff51576d4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732376539652 2024-11-23T15:42:24,093 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 670fe19dfabb4d1fb7773259be9acf61, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732376539741 2024-11-23T15:42:24,093 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting b1c255e554c547cebc23fc0c92849ea1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732376539757 2024-11-23T15:42:24,093 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d8b25e6ebfe42fdb7812c1365f0ca7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732376540958 2024-11-23T15:42:24,093 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c36375129ce4be6a02c1bda02a9ef69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732376540958 2024-11-23T15:42:24,093 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1df7158b04274d1785eaf46c4d7eb735, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732376541686 2024-11-23T15:42:24,093 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 2112ab8e48aa433cb3ff807f6b833369, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732376541690 2024-11-23T15:42:24,102 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#B#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:24,102 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/848b9105c2fb4590a2037bc7434184c4 is 50, key is test_row_0/B:col10/1732376541690/Put/seqid=0 2024-11-23T15:42:24,105 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#A#compaction#403 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:24,106 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/1134531e7fb24c70b0b2a342630c43e1 is 50, key is test_row_0/A:col10/1732376541690/Put/seqid=0 2024-11-23T15:42:24,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742292_1468 (size=12765) 2024-11-23T15:42:24,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742293_1469 (size=12765) 2024-11-23T15:42:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-23T15:42:24,425 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:24,425 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:42:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-23T15:42:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-23T15:42:24,427 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:42:24,427 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:42:24,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:42:24,513 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/848b9105c2fb4590a2037bc7434184c4 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/848b9105c2fb4590a2037bc7434184c4 2024-11-23T15:42:24,513 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/1134531e7fb24c70b0b2a342630c43e1 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/1134531e7fb24c70b0b2a342630c43e1 2024-11-23T15:42:24,516 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2888821c5754525ea6530da454808a84/B of 2888821c5754525ea6530da454808a84 into 848b9105c2fb4590a2037bc7434184c4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:24,516 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2888821c5754525ea6530da454808a84/A of 2888821c5754525ea6530da454808a84 into 1134531e7fb24c70b0b2a342630c43e1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:24,516 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:24,516 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:24,516 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/A, priority=12, startTime=1732376544091; duration=0sec 2024-11-23T15:42:24,516 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/B, priority=12, startTime=1732376544091; duration=0sec 2024-11-23T15:42:24,516 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:24,516 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:A 2024-11-23T15:42:24,516 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:24,516 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:B 2024-11-23T15:42:24,516 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:42:24,517 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49082 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:42:24,517 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/C is initiating minor compaction (all files) 2024-11-23T15:42:24,518 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/C in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:24,518 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/9f2012a7307f4d98ba8a823c2562688e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/63d04a29356949b8b7e963467b929db9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/2e7a08099b604e4ab9e1df8b4b83b752, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/fed8bfc8d3c342b7bf588dc596c77ced] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=47.9 K 2024-11-23T15:42:24,518 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f2012a7307f4d98ba8a823c2562688e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732376539652 2024-11-23T15:42:24,518 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63d04a29356949b8b7e963467b929db9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732376539757 2024-11-23T15:42:24,519 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e7a08099b604e4ab9e1df8b4b83b752, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732376540958 2024-11-23T15:42:24,519 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting fed8bfc8d3c342b7bf588dc596c77ced, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732376541690 2024-11-23T15:42:24,525 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#C#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:24,526 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/62bede9a220b498a88143fe038dd97d7 is 50, key is test_row_0/C:col10/1732376541690/Put/seqid=0 2024-11-23T15:42:24,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742294_1470 (size=12765) 2024-11-23T15:42:24,534 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/62bede9a220b498a88143fe038dd97d7 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/62bede9a220b498a88143fe038dd97d7 2024-11-23T15:42:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-23T15:42:24,537 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2888821c5754525ea6530da454808a84/C of 2888821c5754525ea6530da454808a84 into 62bede9a220b498a88143fe038dd97d7(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:24,537 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:24,537 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/C, priority=12, startTime=1732376544091; duration=0sec 2024-11-23T15:42:24,538 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:24,538 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:C 2024-11-23T15:42:24,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-23T15:42:24,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:24,580 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-23T15:42:24,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/e66b8607722644e29a2a98fc88ac4cb1 is 50, key is test_row_0/A:col10/1732376542872/Put/seqid=0 2024-11-23T15:42:24,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742295_1471 (size=9857) 2024-11-23T15:42:24,587 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/e66b8607722644e29a2a98fc88ac4cb1 2024-11-23T15:42:24,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/1278369398ee481d894fcf5e51ff0cce is 50, key is test_row_0/B:col10/1732376542872/Put/seqid=0 2024-11-23T15:42:24,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742296_1472 (size=9857) 2024-11-23T15:42:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-23T15:42:25,002 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/1278369398ee481d894fcf5e51ff0cce 2024-11-23T15:42:25,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/ae33774dad0d4c03918284aae83ca0fc is 50, key is test_row_0/C:col10/1732376542872/Put/seqid=0 2024-11-23T15:42:25,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742297_1473 (size=9857) 2024-11-23T15:42:25,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 2888821c5754525ea6530da454808a84 2024-11-23T15:42:25,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:25,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-23T15:42:25,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:25,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376605104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:25,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:25,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:25,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:25,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376605110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:25,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:25,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:25,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:25,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:25,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376605224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:25,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:25,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:25,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376605224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:25,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:25,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:25,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:25,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:25,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:25,412 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/ae33774dad0d4c03918284aae83ca0fc 2024-11-23T15:42:25,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/e66b8607722644e29a2a98fc88ac4cb1 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/e66b8607722644e29a2a98fc88ac4cb1 2024-11-23T15:42:25,418 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/e66b8607722644e29a2a98fc88ac4cb1, entries=100, sequenceid=268, filesize=9.6 K 2024-11-23T15:42:25,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/1278369398ee481d894fcf5e51ff0cce as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/1278369398ee481d894fcf5e51ff0cce 2024-11-23T15:42:25,421 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/1278369398ee481d894fcf5e51ff0cce, entries=100, sequenceid=268, filesize=9.6 K 2024-11-23T15:42:25,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/ae33774dad0d4c03918284aae83ca0fc as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/ae33774dad0d4c03918284aae83ca0fc 2024-11-23T15:42:25,425 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/ae33774dad0d4c03918284aae83ca0fc, entries=100, sequenceid=268, filesize=9.6 K 2024-11-23T15:42:25,425 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 2888821c5754525ea6530da454808a84 in 845ms, sequenceid=268, compaction requested=false 2024-11-23T15:42:25,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:25,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:25,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-23T15:42:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-23T15:42:25,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-23T15:42:25,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0000 sec 2024-11-23T15:42:25,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.0030 sec 2024-11-23T15:42:25,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 2888821c5754525ea6530da454808a84 2024-11-23T15:42:25,438 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:25,438 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-23T15:42:25,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/cb315ae09f6c4a49897d08db456b2d74 is 50, key is test_row_0/A:col10/1732376545437/Put/seqid=0 2024-11-23T15:42:25,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742298_1474 (size=14741) 2024-11-23T15:42:25,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/cb315ae09f6c4a49897d08db456b2d74 2024-11-23T15:42:25,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/b3a11e1e37e641868c823f982690f040 is 50, key is test_row_0/B:col10/1732376545437/Put/seqid=0 2024-11-23T15:42:25,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742299_1475 (size=12301) 2024-11-23T15:42:25,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/b3a11e1e37e641868c823f982690f040 2024-11-23T15:42:25,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/276be7e6e2b74712b7b072166bac12cb is 50, key is test_row_0/C:col10/1732376545437/Put/seqid=0 2024-11-23T15:42:25,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742300_1476 (size=12301) 2024-11-23T15:42:25,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376605459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:25,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:25,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376605468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:25,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:25,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:25,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:25,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-23T15:42:25,564 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:25,564 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:42:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-23T15:42:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-23T15:42:25,566 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:42:25,566 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:42:25,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:42:25,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376605574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:25,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:25,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:25,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:25,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376605584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-23T15:42:25,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-23T15:42:25,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:25,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:25,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:25,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:25,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:25,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376605784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:25,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:25,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:25,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:25,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:25,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376605794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:25,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:25,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:25,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:25,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/276be7e6e2b74712b7b072166bac12cb 2024-11-23T15:42:25,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/cb315ae09f6c4a49897d08db456b2d74 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/cb315ae09f6c4a49897d08db456b2d74 2024-11-23T15:42:25,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/cb315ae09f6c4a49897d08db456b2d74, entries=200, sequenceid=295, filesize=14.4 K 2024-11-23T15:42:25,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/b3a11e1e37e641868c823f982690f040 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/b3a11e1e37e641868c823f982690f040 2024-11-23T15:42:25,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-23T15:42:25,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:25,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:25,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:25,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:25,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:25,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:25,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/b3a11e1e37e641868c823f982690f040, entries=150, sequenceid=295, filesize=12.0 K 2024-11-23T15:42:25,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/276be7e6e2b74712b7b072166bac12cb as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/276be7e6e2b74712b7b072166bac12cb 2024-11-23T15:42:25,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/276be7e6e2b74712b7b072166bac12cb, entries=150, sequenceid=295, filesize=12.0 K 2024-11-23T15:42:25,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 2888821c5754525ea6530da454808a84 in 440ms, sequenceid=295, compaction requested=true 2024-11-23T15:42:25,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:25,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:42:25,878 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:25,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:25,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:42:25,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:25,878 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:25,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:42:25,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:25,879 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:25,879 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:25,879 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/A is initiating minor compaction (all files) 2024-11-23T15:42:25,879 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/B is initiating minor compaction (all files) 2024-11-23T15:42:25,879 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/A in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:25,879 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/B in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:25,879 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/1134531e7fb24c70b0b2a342630c43e1, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/e66b8607722644e29a2a98fc88ac4cb1, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/cb315ae09f6c4a49897d08db456b2d74] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=36.5 K 2024-11-23T15:42:25,879 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/848b9105c2fb4590a2037bc7434184c4, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/1278369398ee481d894fcf5e51ff0cce, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/b3a11e1e37e641868c823f982690f040] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=34.1 K 2024-11-23T15:42:25,879 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1134531e7fb24c70b0b2a342630c43e1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732376541690 2024-11-23T15:42:25,879 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 848b9105c2fb4590a2037bc7434184c4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732376541690 2024-11-23T15:42:25,880 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting e66b8607722644e29a2a98fc88ac4cb1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732376542872 2024-11-23T15:42:25,880 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 1278369398ee481d894fcf5e51ff0cce, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732376542872 2024-11-23T15:42:25,880 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb315ae09f6c4a49897d08db456b2d74, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732376545098 2024-11-23T15:42:25,880 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting b3a11e1e37e641868c823f982690f040, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732376545098 2024-11-23T15:42:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-23T15:42:25,885 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#A#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:25,885 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/84d8451f9f6247338f9e879b24dd82b5 is 50, key is test_row_0/A:col10/1732376545437/Put/seqid=0 2024-11-23T15:42:25,885 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#B#compaction#412 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:25,886 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/1300352ad91d42e5b27cdfe14ea148a5 is 50, key is test_row_0/B:col10/1732376545437/Put/seqid=0 2024-11-23T15:42:25,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742302_1478 (size=13017) 2024-11-23T15:42:25,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742301_1477 (size=13017) 2024-11-23T15:42:25,895 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/84d8451f9f6247338f9e879b24dd82b5 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/84d8451f9f6247338f9e879b24dd82b5 2024-11-23T15:42:25,895 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/1300352ad91d42e5b27cdfe14ea148a5 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/1300352ad91d42e5b27cdfe14ea148a5 2024-11-23T15:42:25,900 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2888821c5754525ea6530da454808a84/A of 2888821c5754525ea6530da454808a84 into 84d8451f9f6247338f9e879b24dd82b5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:25,900 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:25,900 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/A, priority=13, startTime=1732376545878; duration=0sec 2024-11-23T15:42:25,900 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:25,900 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:A 2024-11-23T15:42:25,900 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:25,901 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:25,901 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/C is initiating minor compaction (all files) 2024-11-23T15:42:25,901 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/C in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:25,901 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/62bede9a220b498a88143fe038dd97d7, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/ae33774dad0d4c03918284aae83ca0fc, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/276be7e6e2b74712b7b072166bac12cb] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=34.1 K 2024-11-23T15:42:25,901 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62bede9a220b498a88143fe038dd97d7, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732376541690 2024-11-23T15:42:25,902 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae33774dad0d4c03918284aae83ca0fc, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732376542872 2024-11-23T15:42:25,902 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 276be7e6e2b74712b7b072166bac12cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732376545098 2024-11-23T15:42:25,904 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2888821c5754525ea6530da454808a84/B of 2888821c5754525ea6530da454808a84 into 1300352ad91d42e5b27cdfe14ea148a5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:25,904 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:25,904 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/B, priority=13, startTime=1732376545878; duration=0sec 2024-11-23T15:42:25,904 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:25,904 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:B 2024-11-23T15:42:25,909 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#C#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:25,909 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/a8295d4558c64e4ca9bf8f85c2c354fa is 50, key is test_row_0/C:col10/1732376545437/Put/seqid=0 2024-11-23T15:42:25,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742303_1479 (size=13017) 2024-11-23T15:42:25,930 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/a8295d4558c64e4ca9bf8f85c2c354fa as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/a8295d4558c64e4ca9bf8f85c2c354fa 2024-11-23T15:42:25,934 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2888821c5754525ea6530da454808a84/C of 2888821c5754525ea6530da454808a84 into a8295d4558c64e4ca9bf8f85c2c354fa(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:25,934 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:25,934 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/C, priority=13, startTime=1732376545878; duration=0sec 2024-11-23T15:42:25,934 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:25,934 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:C 2024-11-23T15:42:26,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-23T15:42:26,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:26,024 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-23T15:42:26,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/71142fb7065141aead6b7b0a8e3bb500 is 50, key is test_row_0/A:col10/1732376545466/Put/seqid=0 2024-11-23T15:42:26,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742304_1480 (size=12301) 2024-11-23T15:42:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 2888821c5754525ea6530da454808a84 2024-11-23T15:42:26,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:26,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376606104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:26,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,109 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:26,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376606104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:26,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:26,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376606105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:26,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:26,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:26,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376606106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:26,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:26,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:26,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376606109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:26,117 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,117 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,117 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-23T15:42:26,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376606214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:26,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:26,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376606224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:26,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376606224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376606224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:26,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376606424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:26,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:26,431 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/71142fb7065141aead6b7b0a8e3bb500 2024-11-23T15:42:26,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/6451bb55837c42c4b95f503067ed2a76 is 50, key is test_row_0/B:col10/1732376545466/Put/seqid=0 2024-11-23T15:42:26,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376606434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:26,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:26,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376606434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:26,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:26,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376606434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:26,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742305_1481 (size=12301) 2024-11-23T15:42:26,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376606624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:26,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-23T15:42:26,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376606734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:26,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:26,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376606743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:26,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:26,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376606744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:26,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:26,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376606744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:26,844 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/6451bb55837c42c4b95f503067ed2a76 2024-11-23T15:42:26,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/26b6423d4fa74e81afb1853f8f5aa1be is 50, key is test_row_0/C:col10/1732376545466/Put/seqid=0 2024-11-23T15:42:26,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742306_1482 (size=12301) 2024-11-23T15:42:27,255 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/26b6423d4fa74e81afb1853f8f5aa1be 2024-11-23T15:42:27,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/71142fb7065141aead6b7b0a8e3bb500 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/71142fb7065141aead6b7b0a8e3bb500 2024-11-23T15:42:27,261 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/71142fb7065141aead6b7b0a8e3bb500, entries=150, sequenceid=307, filesize=12.0 K 2024-11-23T15:42:27,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376607254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:27,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376607255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:27,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/6451bb55837c42c4b95f503067ed2a76 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/6451bb55837c42c4b95f503067ed2a76 2024-11-23T15:42:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:27,265 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/6451bb55837c42c4b95f503067ed2a76, entries=150, sequenceid=307, filesize=12.0 K 2024-11-23T15:42:27,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/26b6423d4fa74e81afb1853f8f5aa1be as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/26b6423d4fa74e81afb1853f8f5aa1be 2024-11-23T15:42:27,268 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/26b6423d4fa74e81afb1853f8f5aa1be, entries=150, sequenceid=307, filesize=12.0 K 2024-11-23T15:42:27,269 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 2888821c5754525ea6530da454808a84 in 1245ms, sequenceid=307, compaction requested=false 2024-11-23T15:42:27,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:27,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:27,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-23T15:42:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-23T15:42:27,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-23T15:42:27,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7040 sec 2024-11-23T15:42:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 2888821c5754525ea6530da454808a84 2024-11-23T15:42:27,272 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:27,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-23T15:42:27,274 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.7090 sec 2024-11-23T15:42:27,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/1d26997869f9424fa1c78e2ce578c71e is 50, key is test_row_0/A:col10/1732376547271/Put/seqid=0 2024-11-23T15:42:27,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742307_1483 (size=17181) 2024-11-23T15:42:27,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/1d26997869f9424fa1c78e2ce578c71e 2024-11-23T15:42:27,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/49cbb14607294b92b9ffc70aa3aaa4ee is 50, key is test_row_0/B:col10/1732376547271/Put/seqid=0 2024-11-23T15:42:27,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376607290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:27,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:27,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:27,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:27,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376607291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:27,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:27,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742308_1484 (size=12301) 2024-11-23T15:42:27,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/49cbb14607294b92b9ffc70aa3aaa4ee 2024-11-23T15:42:27,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/fb0637e7c50a485698992fa522767e54 is 50, key is test_row_0/C:col10/1732376547271/Put/seqid=0 2024-11-23T15:42:27,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742309_1485 (size=12301) 2024-11-23T15:42:27,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/fb0637e7c50a485698992fa522767e54 2024-11-23T15:42:27,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/1d26997869f9424fa1c78e2ce578c71e as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/1d26997869f9424fa1c78e2ce578c71e 2024-11-23T15:42:27,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/1d26997869f9424fa1c78e2ce578c71e, entries=250, sequenceid=336, filesize=16.8 K 2024-11-23T15:42:27,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/49cbb14607294b92b9ffc70aa3aaa4ee as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/49cbb14607294b92b9ffc70aa3aaa4ee 2024-11-23T15:42:27,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/49cbb14607294b92b9ffc70aa3aaa4ee, entries=150, sequenceid=336, filesize=12.0 K 2024-11-23T15:42:27,322 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-23T15:42:27,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/fb0637e7c50a485698992fa522767e54 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/fb0637e7c50a485698992fa522767e54 2024-11-23T15:42:27,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/fb0637e7c50a485698992fa522767e54, entries=150, sequenceid=336, filesize=12.0 K 2024-11-23T15:42:27,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 2888821c5754525ea6530da454808a84 in 55ms, sequenceid=336, compaction requested=true 2024-11-23T15:42:27,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:27,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:42:27,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:27,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:42:27,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:27,328 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:27,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:42:27,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:27,328 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:27,328 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:27,328 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42499 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:27,328 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/B is initiating minor compaction (all files) 2024-11-23T15:42:27,328 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/A is initiating minor compaction (all files) 2024-11-23T15:42:27,328 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/B in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:27,328 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/A in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:27,328 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/1300352ad91d42e5b27cdfe14ea148a5, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/6451bb55837c42c4b95f503067ed2a76, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/49cbb14607294b92b9ffc70aa3aaa4ee] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=36.7 K 2024-11-23T15:42:27,328 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/84d8451f9f6247338f9e879b24dd82b5, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/71142fb7065141aead6b7b0a8e3bb500, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/1d26997869f9424fa1c78e2ce578c71e] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=41.5 K 2024-11-23T15:42:27,329 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84d8451f9f6247338f9e879b24dd82b5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732376545098 2024-11-23T15:42:27,329 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 1300352ad91d42e5b27cdfe14ea148a5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732376545098 2024-11-23T15:42:27,329 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71142fb7065141aead6b7b0a8e3bb500, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732376545446 2024-11-23T15:42:27,330 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6451bb55837c42c4b95f503067ed2a76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732376545446 2024-11-23T15:42:27,330 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d26997869f9424fa1c78e2ce578c71e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732376546103 2024-11-23T15:42:27,330 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 49cbb14607294b92b9ffc70aa3aaa4ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732376546104 2024-11-23T15:42:27,335 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#A#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:27,335 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#B#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:27,336 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/8752f6ea6454443dbf16c0db37912a7b is 50, key is test_row_0/A:col10/1732376547271/Put/seqid=0 2024-11-23T15:42:27,336 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/d57c49c78a5b4e0392bf9b3a1d755cb8 is 50, key is test_row_0/B:col10/1732376547271/Put/seqid=0 2024-11-23T15:42:27,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742310_1486 (size=13119) 2024-11-23T15:42:27,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742311_1487 (size=13119) 2024-11-23T15:42:27,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 2888821c5754525ea6530da454808a84 2024-11-23T15:42:27,419 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:27,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T15:42:27,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/95c2a821cb65405ab66a50640bc3ba2d is 50, key is test_row_0/A:col10/1732376547418/Put/seqid=0 2024-11-23T15:42:27,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742312_1488 (size=17181) 2024-11-23T15:42:27,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376607547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:27,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376607552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:27,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:27,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:27,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:27,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376607634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:27,641 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_2' in 2888821c5754525ea6530da454808a84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:42:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:27,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376607665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:27,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:27,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376607675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:27,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-23T15:42:27,725 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:27,725 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:42:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-23T15:42:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-23T15:42:27,726 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:42:27,727 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:42:27,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:42:27,749 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/8752f6ea6454443dbf16c0db37912a7b as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/8752f6ea6454443dbf16c0db37912a7b 2024-11-23T15:42:27,752 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/d57c49c78a5b4e0392bf9b3a1d755cb8 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/d57c49c78a5b4e0392bf9b3a1d755cb8 2024-11-23T15:42:27,755 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2888821c5754525ea6530da454808a84/A of 2888821c5754525ea6530da454808a84 into 8752f6ea6454443dbf16c0db37912a7b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:27,755 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:27,755 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/A, priority=13, startTime=1732376547327; duration=0sec 2024-11-23T15:42:27,755 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:27,755 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:A 2024-11-23T15:42:27,755 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:27,757 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:27,757 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/C is initiating minor compaction (all files) 2024-11-23T15:42:27,757 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/C in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:27,757 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/a8295d4558c64e4ca9bf8f85c2c354fa, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/26b6423d4fa74e81afb1853f8f5aa1be, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/fb0637e7c50a485698992fa522767e54] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=36.7 K 2024-11-23T15:42:27,758 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8295d4558c64e4ca9bf8f85c2c354fa, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732376545098 2024-11-23T15:42:27,758 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26b6423d4fa74e81afb1853f8f5aa1be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732376545446 2024-11-23T15:42:27,759 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb0637e7c50a485698992fa522767e54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732376546104 2024-11-23T15:42:27,759 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2888821c5754525ea6530da454808a84/B of 2888821c5754525ea6530da454808a84 into d57c49c78a5b4e0392bf9b3a1d755cb8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:27,759 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:27,759 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/B, priority=13, startTime=1732376547327; duration=0sec 2024-11-23T15:42:27,759 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:27,759 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:B 2024-11-23T15:42:27,765 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#C#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:27,765 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/318fe3c807a84fd382433c68be36a794 is 50, key is test_row_0/C:col10/1732376547271/Put/seqid=0 2024-11-23T15:42:27,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742313_1489 (size=13119) 2024-11-23T15:42:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-23T15:42:27,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/95c2a821cb65405ab66a50640bc3ba2d 2024-11-23T15:42:27,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/9cfa9c3a13a049568f736be583be4495 is 50, key is test_row_0/B:col10/1732376547418/Put/seqid=0 2024-11-23T15:42:27,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742314_1490 (size=12301) 2024-11-23T15:42:27,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-23T15:42:27,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:27,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:27,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:27,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:27,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:27,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376607884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:27,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:27,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:27,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:27,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376607894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:27,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:27,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:27,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:28,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-23T15:42:28,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:28,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:28,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:28,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:28,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-23T15:42:28,173 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/318fe3c807a84fd382433c68be36a794 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/318fe3c807a84fd382433c68be36a794 2024-11-23T15:42:28,176 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2888821c5754525ea6530da454808a84/C of 2888821c5754525ea6530da454808a84 into 318fe3c807a84fd382433c68be36a794(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:28,176 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:28,177 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/C, priority=13, startTime=1732376547328; duration=0sec 2024-11-23T15:42:28,177 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:28,177 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:C 2024-11-23T15:42:28,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-23T15:42:28,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:28,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:28,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:28,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:28,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:28,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376608204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:28,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376608214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:28,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:28,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/9cfa9c3a13a049568f736be583be4495 2024-11-23T15:42:28,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/33fb9cac80b64157990ee93bcb838ba3 is 50, key is test_row_0/C:col10/1732376547418/Put/seqid=0 2024-11-23T15:42:28,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742315_1491 (size=12301) 2024-11-23T15:42:28,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/33fb9cac80b64157990ee93bcb838ba3 2024-11-23T15:42:28,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/95c2a821cb65405ab66a50640bc3ba2d as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/95c2a821cb65405ab66a50640bc3ba2d 2024-11-23T15:42:28,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/95c2a821cb65405ab66a50640bc3ba2d, entries=250, sequenceid=347, filesize=16.8 K 2024-11-23T15:42:28,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/9cfa9c3a13a049568f736be583be4495 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/9cfa9c3a13a049568f736be583be4495 2024-11-23T15:42:28,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/9cfa9c3a13a049568f736be583be4495, entries=150, sequenceid=347, filesize=12.0 K 2024-11-23T15:42:28,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/33fb9cac80b64157990ee93bcb838ba3 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/33fb9cac80b64157990ee93bcb838ba3 2024-11-23T15:42:28,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376608274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:28,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376608274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:28,279 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_0' in 2888821c5754525ea6530da454808a84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:42:28,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:28,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:28,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:28,279 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_1' in 2888821c5754525ea6530da454808a84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:42:28,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:28,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:28,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/33fb9cac80b64157990ee93bcb838ba3, entries=150, sequenceid=347, filesize=12.0 K 2024-11-23T15:42:28,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2888821c5754525ea6530da454808a84 in 864ms, sequenceid=347, compaction requested=false 2024-11-23T15:42:28,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:28,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-23T15:42:28,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:28,339 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T15:42:28,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/ebd00e99b6164f08898eb759d23a3fa8 is 50, key is test_row_0/A:col10/1732376547545/Put/seqid=0 2024-11-23T15:42:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-23T15:42:28,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742316_1492 (size=12301) 2024-11-23T15:42:28,368 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/ebd00e99b6164f08898eb759d23a3fa8 2024-11-23T15:42:28,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/8fc58050f61b42ec924f0be6c3835124 is 50, key is test_row_0/B:col10/1732376547545/Put/seqid=0 2024-11-23T15:42:28,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742317_1493 (size=12301) 2024-11-23T15:42:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 2888821c5754525ea6530da454808a84 2024-11-23T15:42:28,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. as already flushing 2024-11-23T15:42:28,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376608766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:28,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:28,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:28,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:28,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376608767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:28,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:28,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:28,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:28,780 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/8fc58050f61b42ec924f0be6c3835124 2024-11-23T15:42:28,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/66f65237d9e447d5a99684c99c0b633c is 50, key is test_row_0/C:col10/1732376547545/Put/seqid=0 2024-11-23T15:42:28,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742318_1494 (size=12301) 2024-11-23T15:42:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-23T15:42:28,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376608884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:28,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:28,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376608884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:28,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:28,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:28,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:28,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:28,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:29,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376609094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:29,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376609094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:29,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:29,189 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/66f65237d9e447d5a99684c99c0b633c 2024-11-23T15:42:29,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/ebd00e99b6164f08898eb759d23a3fa8 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/ebd00e99b6164f08898eb759d23a3fa8 2024-11-23T15:42:29,195 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/ebd00e99b6164f08898eb759d23a3fa8, entries=150, sequenceid=375, filesize=12.0 K 2024-11-23T15:42:29,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/8fc58050f61b42ec924f0be6c3835124 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8fc58050f61b42ec924f0be6c3835124 2024-11-23T15:42:29,199 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8fc58050f61b42ec924f0be6c3835124, entries=150, sequenceid=375, filesize=12.0 K 2024-11-23T15:42:29,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/66f65237d9e447d5a99684c99c0b633c as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/66f65237d9e447d5a99684c99c0b633c 2024-11-23T15:42:29,204 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/66f65237d9e447d5a99684c99c0b633c, entries=150, sequenceid=375, filesize=12.0 K 2024-11-23T15:42:29,205 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2888821c5754525ea6530da454808a84 in 866ms, sequenceid=375, compaction requested=true 2024-11-23T15:42:29,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:29,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:29,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-23T15:42:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-23T15:42:29,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-23T15:42:29,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4790 sec 2024-11-23T15:42:29,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 1.4830 sec 2024-11-23T15:42:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 2888821c5754525ea6530da454808a84 2024-11-23T15:42:29,419 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:29,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T15:42:29,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/027e7f3fd0014d5aaf6c62f7ef902d59 is 50, key is test_row_0/A:col10/1732376549418/Put/seqid=0 2024-11-23T15:42:29,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742319_1495 (size=17181) 2024-11-23T15:42:29,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376609508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:29,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376609518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:29,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:29,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376609624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:29,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:29,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:29,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:29,635 INFO [Thread-1884 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1884. 2024-11-23T15:42:29,636 DEBUG [Thread-1884 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:29,636 DEBUG [Thread-1884 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:29,636 DEBUG [Thread-1884 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:29,636 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:29,637 INFO [Thread-1878 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1878. 2024-11-23T15:42:29,637 DEBUG [Thread-1878 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:29,637 DEBUG [Thread-1878 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:29,637 DEBUG [Thread-1878 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:29,637 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:29,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376609634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:29,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:29,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:29,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:29,648 INFO [Thread-1886 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1886. 2024-11-23T15:42:29,649 DEBUG [Thread-1886 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:29,649 DEBUG [Thread-1886 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:29,649 DEBUG [Thread-1886 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:29,649 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:29,649 INFO [Thread-1880 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1880. 2024-11-23T15:42:29,649 DEBUG [Thread-1880 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:29,649 DEBUG [Thread-1880 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:29,650 DEBUG [Thread-1880 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:29,650 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:29,651 INFO [Thread-1882 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1882. 2024-11-23T15:42:29,651 DEBUG [Thread-1882 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:29,651 DEBUG [Thread-1882 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:29,651 DEBUG [Thread-1882 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:29,651 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:29,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37168 deadline: 1732376609654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:29,655 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_2' in 2888821c5754525ea6530da454808a84 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:42:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:29,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/027e7f3fd0014d5aaf6c62f7ef902d59 2024-11-23T15:42:29,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:29,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/cbe05a0315f94f5983ca7cd789626750 is 50, key is test_row_0/B:col10/1732376549418/Put/seqid=0 2024-11-23T15:42:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376609834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:29,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742320_1496 (size=12301) 2024-11-23T15:42:29,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:29,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376609854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:29,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:29,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:29,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:29,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-23T15:42:29,875 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:30,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:30,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376610144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:30,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:30,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37176 deadline: 1732376610164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:30,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/cbe05a0315f94f5983ca7cd789626750 2024-11-23T15:42:30,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/e513d9a80a8148248af27e1dffdad44c is 50, key is test_row_0/C:col10/1732376549418/Put/seqid=0 2024-11-23T15:42:30,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742321_1497 (size=12301) 2024-11-23T15:42:30,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:30,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37188 deadline: 1732376610295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37198 deadline: 1732376610295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:30,295 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_1' in 2888821c5754525ea6530da454808a84 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:42:30,295 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_0' in 2888821c5754525ea6530da454808a84 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:42:30,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:30,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:30,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:30,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:30,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:30,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:30,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/e513d9a80a8148248af27e1dffdad44c 2024-11-23T15:42:30,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/027e7f3fd0014d5aaf6c62f7ef902d59 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/027e7f3fd0014d5aaf6c62f7ef902d59 2024-11-23T15:42:30,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:30,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37230 deadline: 1732376610654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:30,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 , the old value is region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:30,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:30,655 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., hostname=b712f9af2c12,46589,1732376409194, seqNum=2 because the exception is null or not the one we care about 2024-11-23T15:42:30,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/027e7f3fd0014d5aaf6c62f7ef902d59, entries=250, sequenceid=388, filesize=16.8 K 2024-11-23T15:42:30,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/cbe05a0315f94f5983ca7cd789626750 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/cbe05a0315f94f5983ca7cd789626750 2024-11-23T15:42:30,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/cbe05a0315f94f5983ca7cd789626750, entries=150, sequenceid=388, filesize=12.0 K 2024-11-23T15:42:30,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/e513d9a80a8148248af27e1dffdad44c as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/e513d9a80a8148248af27e1dffdad44c 2024-11-23T15:42:30,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/e513d9a80a8148248af27e1dffdad44c, entries=150, sequenceid=388, filesize=12.0 K 2024-11-23T15:42:30,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 2888821c5754525ea6530da454808a84 in 1246ms, sequenceid=388, compaction requested=true 2024-11-23T15:42:30,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:30,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:42:30,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:30,665 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:42:30,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:42:30,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:30,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2888821c5754525ea6530da454808a84:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:42:30,666 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:42:30,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:30,667 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 59782 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:42:30,667 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:42:30,667 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/A is initiating minor compaction (all files) 2024-11-23T15:42:30,667 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/B is initiating minor compaction (all files) 2024-11-23T15:42:30,667 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/A in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:30,667 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/B in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:30,667 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/d57c49c78a5b4e0392bf9b3a1d755cb8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/9cfa9c3a13a049568f736be583be4495, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8fc58050f61b42ec924f0be6c3835124, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/cbe05a0315f94f5983ca7cd789626750] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=48.8 K 2024-11-23T15:42:30,667 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/8752f6ea6454443dbf16c0db37912a7b, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/95c2a821cb65405ab66a50640bc3ba2d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/ebd00e99b6164f08898eb759d23a3fa8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/027e7f3fd0014d5aaf6c62f7ef902d59] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=58.4 K 2024-11-23T15:42:30,667 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting d57c49c78a5b4e0392bf9b3a1d755cb8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732376546104 2024-11-23T15:42:30,667 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8752f6ea6454443dbf16c0db37912a7b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732376546104 2024-11-23T15:42:30,668 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95c2a821cb65405ab66a50640bc3ba2d, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732376547289 2024-11-23T15:42:30,668 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cfa9c3a13a049568f736be583be4495, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732376547408 2024-11-23T15:42:30,668 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebd00e99b6164f08898eb759d23a3fa8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732376547531 2024-11-23T15:42:30,668 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fc58050f61b42ec924f0be6c3835124, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732376547531 2024-11-23T15:42:30,669 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting cbe05a0315f94f5983ca7cd789626750, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732376548765 2024-11-23T15:42:30,669 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 027e7f3fd0014d5aaf6c62f7ef902d59, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732376548739 2024-11-23T15:42:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on 2888821c5754525ea6530da454808a84 2024-11-23T15:42:30,676 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:30,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T15:42:30,676 INFO [Thread-1871 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1871. 2024-11-23T15:42:30,676 DEBUG [Thread-1871 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:30,676 DEBUG [Thread-1871 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:30,676 DEBUG [Thread-1871 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:30,676 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:30,679 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#B#compaction#432 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:30,679 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/118749842cc147d598787e733eb7b7aa is 50, key is test_row_0/B:col10/1732376549418/Put/seqid=0 2024-11-23T15:42:30,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742322_1498 (size=13255) 2024-11-23T15:42:30,688 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/118749842cc147d598787e733eb7b7aa as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/118749842cc147d598787e733eb7b7aa 2024-11-23T15:42:30,693 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2888821c5754525ea6530da454808a84/B of 2888821c5754525ea6530da454808a84 into 118749842cc147d598787e733eb7b7aa(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:30,693 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:30,693 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/B, priority=12, startTime=1732376550665; duration=0sec 2024-11-23T15:42:30,693 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:30,694 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:B 2024-11-23T15:42:30,694 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T15:42:30,695 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T15:42:30,695 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): 2888821c5754525ea6530da454808a84/C is initiating minor compaction (all files) 2024-11-23T15:42:30,695 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2888821c5754525ea6530da454808a84/C in TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:30,695 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/318fe3c807a84fd382433c68be36a794, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/33fb9cac80b64157990ee93bcb838ba3, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/66f65237d9e447d5a99684c99c0b633c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/e513d9a80a8148248af27e1dffdad44c] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp, totalSize=48.8 K 2024-11-23T15:42:30,695 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 318fe3c807a84fd382433c68be36a794, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732376546104 2024-11-23T15:42:30,695 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 33fb9cac80b64157990ee93bcb838ba3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1732376547408 2024-11-23T15:42:30,696 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 66f65237d9e447d5a99684c99c0b633c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1732376547531 2024-11-23T15:42:30,696 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting e513d9a80a8148248af27e1dffdad44c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732376548765 2024-11-23T15:42:30,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/5813055c7bbf4c7e92ccc9b4447fe286 is 50, key is test_row_0/A:col10/1732376549507/Put/seqid=0 2024-11-23T15:42:30,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742323_1499 (size=12301) 2024-11-23T15:42:30,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/5813055c7bbf4c7e92ccc9b4447fe286 2024-11-23T15:42:30,718 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#C#compaction#435 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:30,719 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/47dfc6a1f9c840e99f6425b466aa610d is 50, key is test_row_0/C:col10/1732376549418/Put/seqid=0 2024-11-23T15:42:30,726 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2888821c5754525ea6530da454808a84#A#compaction#433 average throughput is 0.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:30,726 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/f9ff9263f7dd4302960ba643a7b4f62b is 50, key is test_row_0/A:col10/1732376549418/Put/seqid=0 2024-11-23T15:42:30,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742324_1500 (size=13255) 2024-11-23T15:42:30,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742325_1501 (size=13255) 2024-11-23T15:42:30,736 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/47dfc6a1f9c840e99f6425b466aa610d as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/47dfc6a1f9c840e99f6425b466aa610d 2024-11-23T15:42:30,737 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/f9ff9263f7dd4302960ba643a7b4f62b as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/f9ff9263f7dd4302960ba643a7b4f62b 2024-11-23T15:42:30,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/62eef447249b4ffe90d5fa743726b1d5 is 50, key is test_row_0/B:col10/1732376549507/Put/seqid=0 2024-11-23T15:42:30,741 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2888821c5754525ea6530da454808a84/C of 2888821c5754525ea6530da454808a84 into 47dfc6a1f9c840e99f6425b466aa610d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:30,741 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:30,741 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/C, priority=12, startTime=1732376550666; duration=0sec 2024-11-23T15:42:30,742 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:30,742 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:C 2024-11-23T15:42:30,742 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2888821c5754525ea6530da454808a84/A of 2888821c5754525ea6530da454808a84 into f9ff9263f7dd4302960ba643a7b4f62b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:30,742 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:30,742 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., storeName=2888821c5754525ea6530da454808a84/A, priority=12, startTime=1732376550665; duration=0sec 2024-11-23T15:42:30,742 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:30,742 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2888821c5754525ea6530da454808a84:A 2024-11-23T15:42:30,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742326_1502 (size=12301) 2024-11-23T15:42:31,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/62eef447249b4ffe90d5fa743726b1d5 2024-11-23T15:42:31,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/e6d84af8aa43475194f0f054d8594913 is 50, key is test_row_0/C:col10/1732376549507/Put/seqid=0 2024-11-23T15:42:31,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742327_1503 (size=12301) 2024-11-23T15:42:31,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/e6d84af8aa43475194f0f054d8594913 2024-11-23T15:42:31,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/5813055c7bbf4c7e92ccc9b4447fe286 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/5813055c7bbf4c7e92ccc9b4447fe286 2024-11-23T15:42:31,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/5813055c7bbf4c7e92ccc9b4447fe286, entries=150, sequenceid=412, filesize=12.0 K 2024-11-23T15:42:31,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/62eef447249b4ffe90d5fa743726b1d5 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/62eef447249b4ffe90d5fa743726b1d5 2024-11-23T15:42:31,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/62eef447249b4ffe90d5fa743726b1d5, entries=150, sequenceid=412, filesize=12.0 K 2024-11-23T15:42:31,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/e6d84af8aa43475194f0f054d8594913 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/e6d84af8aa43475194f0f054d8594913 2024-11-23T15:42:31,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/e6d84af8aa43475194f0f054d8594913, entries=150, sequenceid=412, filesize=12.0 K 2024-11-23T15:42:31,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 2888821c5754525ea6530da454808a84 in 895ms, sequenceid=412, compaction requested=false 2024-11-23T15:42:31,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2888821c5754525ea6530da454808a84: 2024-11-23T15:42:31,665 INFO [Thread-1875 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1875. 2024-11-23T15:42:31,665 DEBUG [Thread-1875 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:31,665 DEBUG [Thread-1875 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:31,665 DEBUG [Thread-1875 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:31,665 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:33,667 INFO [Thread-1873 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1873. 2024-11-23T15:42:33,667 DEBUG [Thread-1873 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:33,667 DEBUG [Thread-1873 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:33,667 DEBUG [Thread-1873 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:33,667 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:34,308 INFO [Thread-1867 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1867. 2024-11-23T15:42:34,308 DEBUG [Thread-1867 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:34,309 DEBUG [Thread-1867 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:34,309 DEBUG [Thread-1867 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:34,309 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:34,349 INFO [Thread-1869 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1869. 2024-11-23T15:42:34,349 DEBUG [Thread-1869 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:34,349 DEBUG [Thread-1869 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:34,349 DEBUG [Thread-1869 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:34,349 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:34,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T15:42:34,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-23T15:42:34,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-23T15:42:34,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-23T15:42:34,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-23T15:42:34,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-23T15:42:34,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T15:42:34,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T15:42:34,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1628 2024-11-23T15:42:34,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4882 rows 2024-11-23T15:42:34,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1632 2024-11-23T15:42:34,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4888 rows 2024-11-23T15:42:34,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1625 2024-11-23T15:42:34,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4873 rows 2024-11-23T15:42:34,350 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1626 2024-11-23T15:42:34,351 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4876 rows 2024-11-23T15:42:34,351 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1622 2024-11-23T15:42:34,351 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4861 rows 2024-11-23T15:42:34,351 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T15:42:34,351 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T15:42:34,351 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:34,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:34,352 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:34,353 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-23T15:42:34,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T15:42:34,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-23T15:42:34,357 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376554357"}]},"ts":"1732376554357"} 2024-11-23T15:42:34,359 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T15:42:34,359 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T15:42:34,360 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T15:42:34,361 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2888821c5754525ea6530da454808a84, UNASSIGN}] 2024-11-23T15:42:34,362 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2888821c5754525ea6530da454808a84, UNASSIGN 2024-11-23T15:42:34,363 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=2888821c5754525ea6530da454808a84, regionState=CLOSING, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:34,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2888821c5754525ea6530da454808a84, UNASSIGN because future has completed 2024-11-23T15:42:34,364 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T15:42:34,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194}] 2024-11-23T15:42:34,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-23T15:42:34,518 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(122): Close 2888821c5754525ea6530da454808a84 2024-11-23T15:42:34,518 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T15:42:34,519 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1722): Closing 2888821c5754525ea6530da454808a84, disabling compactions & flushes 2024-11-23T15:42:34,519 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:34,519 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:34,519 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. after waiting 0 ms 2024-11-23T15:42:34,519 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:34,519 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(2902): Flushing 2888821c5754525ea6530da454808a84 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-23T15:42:34,526 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/8dfe0d62bd5e4a2993eee77bb0209209 is 50, key is test_row_0/A:col10/1732376554345/Put/seqid=0 2024-11-23T15:42:34,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742328_1504 (size=12301) 2024-11-23T15:42:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-23T15:42:34,933 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/8dfe0d62bd5e4a2993eee77bb0209209 2024-11-23T15:42:34,945 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/27584577784f4731a8b4def61fe240fd is 50, key is test_row_0/B:col10/1732376554345/Put/seqid=0 2024-11-23T15:42:34,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742329_1505 (size=12301) 2024-11-23T15:42:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-23T15:42:35,351 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/27584577784f4731a8b4def61fe240fd 2024-11-23T15:42:35,365 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/ca68ec5d5dc141a381eb12d52809f284 is 50, key is test_row_0/C:col10/1732376554345/Put/seqid=0 2024-11-23T15:42:35,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742330_1506 (size=12301) 2024-11-23T15:42:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-23T15:42:35,770 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/ca68ec5d5dc141a381eb12d52809f284 2024-11-23T15:42:35,780 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/A/8dfe0d62bd5e4a2993eee77bb0209209 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/8dfe0d62bd5e4a2993eee77bb0209209 2024-11-23T15:42:35,783 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/8dfe0d62bd5e4a2993eee77bb0209209, entries=150, sequenceid=422, filesize=12.0 K 2024-11-23T15:42:35,784 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/B/27584577784f4731a8b4def61fe240fd as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/27584577784f4731a8b4def61fe240fd 2024-11-23T15:42:35,787 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/27584577784f4731a8b4def61fe240fd, entries=150, sequenceid=422, filesize=12.0 K 2024-11-23T15:42:35,788 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/.tmp/C/ca68ec5d5dc141a381eb12d52809f284 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/ca68ec5d5dc141a381eb12d52809f284 2024-11-23T15:42:35,791 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/ca68ec5d5dc141a381eb12d52809f284, entries=150, sequenceid=422, filesize=12.0 K 2024-11-23T15:42:35,792 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 2888821c5754525ea6530da454808a84 in 1273ms, sequenceid=422, compaction requested=true 2024-11-23T15:42:35,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/a8b8895d80e445ec82d02bc18dff6e29, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/f527f4816ce240b1a8c56c5103490ff0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/5bbd4e0e8bdc4f5e873c944a1b9e9239, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/93636576e19d4b2081d5e076a901d929, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/d2fd25cff7e84347939dade18569e4d9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/7307937c4de14213a207e23df0836211, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/aedd0a7ec0ed48f788549abb637d2186, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/fcf32fd8df75498eaf73c371a740823f, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/c31c8318944043dfb309dad19066f010, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/0754e376450340f8abf0adef5838ae81, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/dd2d67c5f0d54e90aea809116f04f16b, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/310bde306a98481f9432296cb9e82a40, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/2db7138e86304d13959a7ba64622b225, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/5702fde3f72542c5b747a5ade3cd016c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/78a8dc2870a74412964c91bbb8008210, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/670fe19dfabb4d1fb7773259be9acf61, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/9d8b25e6ebfe42fdb7812c1365f0ca7c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/1df7158b04274d1785eaf46c4d7eb735, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/1134531e7fb24c70b0b2a342630c43e1, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/e66b8607722644e29a2a98fc88ac4cb1, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/cb315ae09f6c4a49897d08db456b2d74, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/84d8451f9f6247338f9e879b24dd82b5, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/71142fb7065141aead6b7b0a8e3bb500, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/1d26997869f9424fa1c78e2ce578c71e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/8752f6ea6454443dbf16c0db37912a7b, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/95c2a821cb65405ab66a50640bc3ba2d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/ebd00e99b6164f08898eb759d23a3fa8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/027e7f3fd0014d5aaf6c62f7ef902d59] to archive 2024-11-23T15:42:35,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T15:42:35,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/a8b8895d80e445ec82d02bc18dff6e29 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/a8b8895d80e445ec82d02bc18dff6e29 2024-11-23T15:42:35,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/f527f4816ce240b1a8c56c5103490ff0 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/f527f4816ce240b1a8c56c5103490ff0 2024-11-23T15:42:35,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/5bbd4e0e8bdc4f5e873c944a1b9e9239 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/5bbd4e0e8bdc4f5e873c944a1b9e9239 2024-11-23T15:42:35,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/93636576e19d4b2081d5e076a901d929 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/93636576e19d4b2081d5e076a901d929 2024-11-23T15:42:35,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/d2fd25cff7e84347939dade18569e4d9 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/d2fd25cff7e84347939dade18569e4d9 2024-11-23T15:42:35,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/7307937c4de14213a207e23df0836211 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/7307937c4de14213a207e23df0836211 2024-11-23T15:42:35,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/aedd0a7ec0ed48f788549abb637d2186 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/aedd0a7ec0ed48f788549abb637d2186 2024-11-23T15:42:35,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/fcf32fd8df75498eaf73c371a740823f to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/fcf32fd8df75498eaf73c371a740823f 2024-11-23T15:42:35,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/c31c8318944043dfb309dad19066f010 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/c31c8318944043dfb309dad19066f010 2024-11-23T15:42:35,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/0754e376450340f8abf0adef5838ae81 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/0754e376450340f8abf0adef5838ae81 2024-11-23T15:42:35,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/dd2d67c5f0d54e90aea809116f04f16b to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/dd2d67c5f0d54e90aea809116f04f16b 2024-11-23T15:42:35,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/310bde306a98481f9432296cb9e82a40 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/310bde306a98481f9432296cb9e82a40 2024-11-23T15:42:35,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/2db7138e86304d13959a7ba64622b225 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/2db7138e86304d13959a7ba64622b225 2024-11-23T15:42:35,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/5702fde3f72542c5b747a5ade3cd016c to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/5702fde3f72542c5b747a5ade3cd016c 2024-11-23T15:42:35,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/78a8dc2870a74412964c91bbb8008210 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/78a8dc2870a74412964c91bbb8008210 2024-11-23T15:42:35,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/670fe19dfabb4d1fb7773259be9acf61 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/670fe19dfabb4d1fb7773259be9acf61 2024-11-23T15:42:35,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/9d8b25e6ebfe42fdb7812c1365f0ca7c to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/9d8b25e6ebfe42fdb7812c1365f0ca7c 2024-11-23T15:42:35,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/1df7158b04274d1785eaf46c4d7eb735 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/1df7158b04274d1785eaf46c4d7eb735 2024-11-23T15:42:35,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/1134531e7fb24c70b0b2a342630c43e1 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/1134531e7fb24c70b0b2a342630c43e1 2024-11-23T15:42:35,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/e66b8607722644e29a2a98fc88ac4cb1 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/e66b8607722644e29a2a98fc88ac4cb1 2024-11-23T15:42:35,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/cb315ae09f6c4a49897d08db456b2d74 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/cb315ae09f6c4a49897d08db456b2d74 2024-11-23T15:42:35,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/84d8451f9f6247338f9e879b24dd82b5 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/84d8451f9f6247338f9e879b24dd82b5 2024-11-23T15:42:35,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/71142fb7065141aead6b7b0a8e3bb500 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/71142fb7065141aead6b7b0a8e3bb500 2024-11-23T15:42:35,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/1d26997869f9424fa1c78e2ce578c71e to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/1d26997869f9424fa1c78e2ce578c71e 2024-11-23T15:42:35,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/8752f6ea6454443dbf16c0db37912a7b to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/8752f6ea6454443dbf16c0db37912a7b 2024-11-23T15:42:35,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/95c2a821cb65405ab66a50640bc3ba2d to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/95c2a821cb65405ab66a50640bc3ba2d 2024-11-23T15:42:35,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/ebd00e99b6164f08898eb759d23a3fa8 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/ebd00e99b6164f08898eb759d23a3fa8 2024-11-23T15:42:35,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/027e7f3fd0014d5aaf6c62f7ef902d59 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/027e7f3fd0014d5aaf6c62f7ef902d59 2024-11-23T15:42:35,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8c99019424494fafb34a01e6779d007a, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/5c78db8c7c9d4b58a8882fceb326c1f1, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/689fb86a242c4d219d7f9031630f211c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/04188a02beff4f5ba45f7cf762c38e97, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/5b583036c6e7457ca493205649473964, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/4d0eb928fbc643218ba4ebc419aee39e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/628453d35a6e442ebd3235308bb59592, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/150a99772b464d558ebe315d4db1f67c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/2f50c842eae84416abfe5a2f275edb31, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/eba2506262d34dd3b258c9357c4e6503, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/24d4dc652d234205a85d1f6415b911a4, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/e6db866b55a642e5ab3f39c5d0fe81e9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/0d77402f495747b6b218679067555b27, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8a2e18662e5b4f7fa229cb3ff51576d4, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/d77c23e4b0034c2e80e6dfb11c4c2947, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/b1c255e554c547cebc23fc0c92849ea1, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8c36375129ce4be6a02c1bda02a9ef69, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/848b9105c2fb4590a2037bc7434184c4, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/2112ab8e48aa433cb3ff807f6b833369, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/1278369398ee481d894fcf5e51ff0cce, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/1300352ad91d42e5b27cdfe14ea148a5, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/b3a11e1e37e641868c823f982690f040, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/6451bb55837c42c4b95f503067ed2a76, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/d57c49c78a5b4e0392bf9b3a1d755cb8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/49cbb14607294b92b9ffc70aa3aaa4ee, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/9cfa9c3a13a049568f736be583be4495, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8fc58050f61b42ec924f0be6c3835124, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/cbe05a0315f94f5983ca7cd789626750] to archive 2024-11-23T15:42:35,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T15:42:35,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8c99019424494fafb34a01e6779d007a to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8c99019424494fafb34a01e6779d007a 2024-11-23T15:42:35,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/5c78db8c7c9d4b58a8882fceb326c1f1 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/5c78db8c7c9d4b58a8882fceb326c1f1 2024-11-23T15:42:35,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/689fb86a242c4d219d7f9031630f211c to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/689fb86a242c4d219d7f9031630f211c 2024-11-23T15:42:35,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/04188a02beff4f5ba45f7cf762c38e97 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/04188a02beff4f5ba45f7cf762c38e97 2024-11-23T15:42:35,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/5b583036c6e7457ca493205649473964 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/5b583036c6e7457ca493205649473964 2024-11-23T15:42:35,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/4d0eb928fbc643218ba4ebc419aee39e to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/4d0eb928fbc643218ba4ebc419aee39e 2024-11-23T15:42:35,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/628453d35a6e442ebd3235308bb59592 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/628453d35a6e442ebd3235308bb59592 2024-11-23T15:42:35,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/150a99772b464d558ebe315d4db1f67c to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/150a99772b464d558ebe315d4db1f67c 2024-11-23T15:42:35,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/2f50c842eae84416abfe5a2f275edb31 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/2f50c842eae84416abfe5a2f275edb31 2024-11-23T15:42:35,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/eba2506262d34dd3b258c9357c4e6503 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/eba2506262d34dd3b258c9357c4e6503 2024-11-23T15:42:35,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/24d4dc652d234205a85d1f6415b911a4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/24d4dc652d234205a85d1f6415b911a4 2024-11-23T15:42:35,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/e6db866b55a642e5ab3f39c5d0fe81e9 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/e6db866b55a642e5ab3f39c5d0fe81e9 2024-11-23T15:42:35,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/0d77402f495747b6b218679067555b27 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/0d77402f495747b6b218679067555b27 2024-11-23T15:42:35,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8a2e18662e5b4f7fa229cb3ff51576d4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8a2e18662e5b4f7fa229cb3ff51576d4 2024-11-23T15:42:35,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/d77c23e4b0034c2e80e6dfb11c4c2947 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/d77c23e4b0034c2e80e6dfb11c4c2947 2024-11-23T15:42:35,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/b1c255e554c547cebc23fc0c92849ea1 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/b1c255e554c547cebc23fc0c92849ea1 2024-11-23T15:42:35,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8c36375129ce4be6a02c1bda02a9ef69 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8c36375129ce4be6a02c1bda02a9ef69 2024-11-23T15:42:35,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/848b9105c2fb4590a2037bc7434184c4 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/848b9105c2fb4590a2037bc7434184c4 2024-11-23T15:42:35,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/2112ab8e48aa433cb3ff807f6b833369 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/2112ab8e48aa433cb3ff807f6b833369 2024-11-23T15:42:35,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/1278369398ee481d894fcf5e51ff0cce to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/1278369398ee481d894fcf5e51ff0cce 2024-11-23T15:42:35,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/1300352ad91d42e5b27cdfe14ea148a5 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/1300352ad91d42e5b27cdfe14ea148a5 2024-11-23T15:42:35,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/b3a11e1e37e641868c823f982690f040 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/b3a11e1e37e641868c823f982690f040 2024-11-23T15:42:35,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/6451bb55837c42c4b95f503067ed2a76 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/6451bb55837c42c4b95f503067ed2a76 2024-11-23T15:42:35,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/d57c49c78a5b4e0392bf9b3a1d755cb8 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/d57c49c78a5b4e0392bf9b3a1d755cb8 2024-11-23T15:42:35,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/49cbb14607294b92b9ffc70aa3aaa4ee to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/49cbb14607294b92b9ffc70aa3aaa4ee 2024-11-23T15:42:35,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/9cfa9c3a13a049568f736be583be4495 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/9cfa9c3a13a049568f736be583be4495 2024-11-23T15:42:35,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8fc58050f61b42ec924f0be6c3835124 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/8fc58050f61b42ec924f0be6c3835124 2024-11-23T15:42:35,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/cbe05a0315f94f5983ca7cd789626750 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/cbe05a0315f94f5983ca7cd789626750 2024-11-23T15:42:35,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/aa9e9463ad9c43c1bd507bc02091c717, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/ece009d56f2444e79f098b9cddf8c9f0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/3e17343bf7124cabba028128ad793412, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/e2694929361f49e7a7337297c61bc4f6, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b04be6f8f6b94c1fac571493d2df4b4e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b7b5fde24323443080f83f5747236e19, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/8832ef1a867a487db41d560089cc4978, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/52df32508cb04a748b7ca15cf1f6e343, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b66a1c8d17344a47938c8d0d3983cb4a, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/823f8a67536c4083bca2717f341d595a, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/7ce25bf8a9d2457fa2ff4ee9cc191c35, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/0a90da16991148c8b6d977c173276869, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/2ce6a9f541014b7dbe6256c618a76b8a, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/9f2012a7307f4d98ba8a823c2562688e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b529406f53e443f7a70a3bc5b3650ec6, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/63d04a29356949b8b7e963467b929db9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/2e7a08099b604e4ab9e1df8b4b83b752, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/62bede9a220b498a88143fe038dd97d7, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/fed8bfc8d3c342b7bf588dc596c77ced, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/ae33774dad0d4c03918284aae83ca0fc, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/a8295d4558c64e4ca9bf8f85c2c354fa, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/276be7e6e2b74712b7b072166bac12cb, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/26b6423d4fa74e81afb1853f8f5aa1be, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/318fe3c807a84fd382433c68be36a794, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/fb0637e7c50a485698992fa522767e54, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/33fb9cac80b64157990ee93bcb838ba3, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/66f65237d9e447d5a99684c99c0b633c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/e513d9a80a8148248af27e1dffdad44c] to archive 2024-11-23T15:42:35,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T15:42:35,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/aa9e9463ad9c43c1bd507bc02091c717 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/aa9e9463ad9c43c1bd507bc02091c717 2024-11-23T15:42:35,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/ece009d56f2444e79f098b9cddf8c9f0 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/ece009d56f2444e79f098b9cddf8c9f0 2024-11-23T15:42:35,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/3e17343bf7124cabba028128ad793412 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/3e17343bf7124cabba028128ad793412 2024-11-23T15:42:35,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/e2694929361f49e7a7337297c61bc4f6 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/e2694929361f49e7a7337297c61bc4f6 2024-11-23T15:42:35,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b04be6f8f6b94c1fac571493d2df4b4e to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b04be6f8f6b94c1fac571493d2df4b4e 2024-11-23T15:42:35,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b7b5fde24323443080f83f5747236e19 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b7b5fde24323443080f83f5747236e19 2024-11-23T15:42:35,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/8832ef1a867a487db41d560089cc4978 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/8832ef1a867a487db41d560089cc4978 2024-11-23T15:42:35,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/52df32508cb04a748b7ca15cf1f6e343 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/52df32508cb04a748b7ca15cf1f6e343 2024-11-23T15:42:35,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b66a1c8d17344a47938c8d0d3983cb4a to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b66a1c8d17344a47938c8d0d3983cb4a 2024-11-23T15:42:35,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/823f8a67536c4083bca2717f341d595a to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/823f8a67536c4083bca2717f341d595a 2024-11-23T15:42:35,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/7ce25bf8a9d2457fa2ff4ee9cc191c35 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/7ce25bf8a9d2457fa2ff4ee9cc191c35 2024-11-23T15:42:35,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/0a90da16991148c8b6d977c173276869 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/0a90da16991148c8b6d977c173276869 2024-11-23T15:42:35,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/2ce6a9f541014b7dbe6256c618a76b8a to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/2ce6a9f541014b7dbe6256c618a76b8a 2024-11-23T15:42:35,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/9f2012a7307f4d98ba8a823c2562688e to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/9f2012a7307f4d98ba8a823c2562688e 2024-11-23T15:42:35,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b529406f53e443f7a70a3bc5b3650ec6 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/b529406f53e443f7a70a3bc5b3650ec6 2024-11-23T15:42:35,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/63d04a29356949b8b7e963467b929db9 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/63d04a29356949b8b7e963467b929db9 2024-11-23T15:42:35,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/2e7a08099b604e4ab9e1df8b4b83b752 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/2e7a08099b604e4ab9e1df8b4b83b752 2024-11-23T15:42:35,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/62bede9a220b498a88143fe038dd97d7 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/62bede9a220b498a88143fe038dd97d7 2024-11-23T15:42:35,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/fed8bfc8d3c342b7bf588dc596c77ced to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/fed8bfc8d3c342b7bf588dc596c77ced 2024-11-23T15:42:35,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/ae33774dad0d4c03918284aae83ca0fc to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/ae33774dad0d4c03918284aae83ca0fc 2024-11-23T15:42:35,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/a8295d4558c64e4ca9bf8f85c2c354fa to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/a8295d4558c64e4ca9bf8f85c2c354fa 2024-11-23T15:42:35,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/276be7e6e2b74712b7b072166bac12cb to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/276be7e6e2b74712b7b072166bac12cb 2024-11-23T15:42:35,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/26b6423d4fa74e81afb1853f8f5aa1be to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/26b6423d4fa74e81afb1853f8f5aa1be 2024-11-23T15:42:35,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/318fe3c807a84fd382433c68be36a794 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/318fe3c807a84fd382433c68be36a794 2024-11-23T15:42:35,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/fb0637e7c50a485698992fa522767e54 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/fb0637e7c50a485698992fa522767e54 2024-11-23T15:42:35,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/33fb9cac80b64157990ee93bcb838ba3 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/33fb9cac80b64157990ee93bcb838ba3 2024-11-23T15:42:35,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/66f65237d9e447d5a99684c99c0b633c to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/66f65237d9e447d5a99684c99c0b633c 2024-11-23T15:42:35,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/e513d9a80a8148248af27e1dffdad44c to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/e513d9a80a8148248af27e1dffdad44c 2024-11-23T15:42:35,877 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/recovered.edits/425.seqid, newMaxSeqId=425, maxSeqId=1 2024-11-23T15:42:35,878 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. 2024-11-23T15:42:35,878 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] regionserver.HRegion(1676): Region close journal for 2888821c5754525ea6530da454808a84: Waiting for close lock at 1732376554518Running coprocessor pre-close hooks at 1732376554518Disabling compacts and flushes for region at 1732376554518Disabling writes for close at 1732376554519 (+1 ms)Obtaining lock to block concurrent updates at 1732376554519Preparing flush snapshotting stores in 2888821c5754525ea6530da454808a84 at 1732376554519Finished memstore snapshotting TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732376554520 (+1 ms)Flushing stores of TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84. at 1732376554521 (+1 ms)Flushing 2888821c5754525ea6530da454808a84/A: creating writer at 1732376554521Flushing 2888821c5754525ea6530da454808a84/A: appending metadata at 1732376554526 (+5 ms)Flushing 2888821c5754525ea6530da454808a84/A: closing flushed file at 1732376554526Flushing 2888821c5754525ea6530da454808a84/B: creating writer at 1732376554942 (+416 ms)Flushing 2888821c5754525ea6530da454808a84/B: appending metadata at 1732376554945 (+3 ms)Flushing 2888821c5754525ea6530da454808a84/B: closing flushed file at 1732376554945Flushing 2888821c5754525ea6530da454808a84/C: creating writer at 1732376555361 (+416 ms)Flushing 2888821c5754525ea6530da454808a84/C: appending metadata at 1732376555364 (+3 ms)Flushing 2888821c5754525ea6530da454808a84/C: closing flushed file at 1732376555364Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1367f424: reopening flushed file at 1732376555779 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2063a58e: reopening flushed file at 1732376555784 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f48dbdf: reopening flushed file at 1732376555787 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 2888821c5754525ea6530da454808a84 in 1273ms, sequenceid=422, compaction requested=true at 1732376555792 (+5 ms)Writing region close event to WAL at 1732376555875 (+83 ms)Running coprocessor post-close hooks at 1732376555877 (+2 ms)Closed at 1732376555878 (+1 ms) 2024-11-23T15:42:35,879 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=146}] handler.UnassignRegionHandler(157): Closed 2888821c5754525ea6530da454808a84 2024-11-23T15:42:35,880 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=2888821c5754525ea6530da454808a84, regionState=CLOSED 2024-11-23T15:42:35,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 because future has completed 2024-11-23T15:42:35,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-23T15:42:35,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; CloseRegionProcedure 2888821c5754525ea6530da454808a84, server=b712f9af2c12,46589,1732376409194 in 1.5170 sec 2024-11-23T15:42:35,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-23T15:42:35,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2888821c5754525ea6530da454808a84, UNASSIGN in 1.5220 sec 2024-11-23T15:42:35,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-23T15:42:35,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5240 sec 2024-11-23T15:42:35,886 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376555886"}]},"ts":"1732376555886"} 2024-11-23T15:42:35,888 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T15:42:35,888 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T15:42:35,890 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5350 sec 2024-11-23T15:42:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-23T15:42:36,505 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:36,507 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-23T15:42:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:42:36,511 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=147, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:42:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-23T15:42:36,513 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=147, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:42:36,519 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84 2024-11-23T15:42:36,522 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A, FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B, FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C, FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/recovered.edits] 2024-11-23T15:42:36,525 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/5813055c7bbf4c7e92ccc9b4447fe286 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/5813055c7bbf4c7e92ccc9b4447fe286 2024-11-23T15:42:36,526 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/8dfe0d62bd5e4a2993eee77bb0209209 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/8dfe0d62bd5e4a2993eee77bb0209209 2024-11-23T15:42:36,527 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/f9ff9263f7dd4302960ba643a7b4f62b to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/A/f9ff9263f7dd4302960ba643a7b4f62b 2024-11-23T15:42:36,529 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/118749842cc147d598787e733eb7b7aa to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/118749842cc147d598787e733eb7b7aa 2024-11-23T15:42:36,529 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/27584577784f4731a8b4def61fe240fd to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/27584577784f4731a8b4def61fe240fd 2024-11-23T15:42:36,530 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/62eef447249b4ffe90d5fa743726b1d5 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/B/62eef447249b4ffe90d5fa743726b1d5 2024-11-23T15:42:36,532 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/47dfc6a1f9c840e99f6425b466aa610d to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/47dfc6a1f9c840e99f6425b466aa610d 2024-11-23T15:42:36,532 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/ca68ec5d5dc141a381eb12d52809f284 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/ca68ec5d5dc141a381eb12d52809f284 2024-11-23T15:42:36,533 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/e6d84af8aa43475194f0f054d8594913 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/C/e6d84af8aa43475194f0f054d8594913 2024-11-23T15:42:36,535 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/recovered.edits/425.seqid to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84/recovered.edits/425.seqid 2024-11-23T15:42:36,536 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/2888821c5754525ea6530da454808a84 2024-11-23T15:42:36,536 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T15:42:36,538 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=147, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:42:36,540 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T15:42:36,541 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T15:42:36,542 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=147, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:42:36,542 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T15:42:36,542 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732376556542"}]},"ts":"9223372036854775807"} 2024-11-23T15:42:36,544 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T15:42:36,544 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2888821c5754525ea6530da454808a84, NAME => 'TestAcidGuarantees,,1732376527437.2888821c5754525ea6530da454808a84.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T15:42:36,544 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T15:42:36,544 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732376556544"}]},"ts":"9223372036854775807"} 2024-11-23T15:42:36,545 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T15:42:36,546 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=147, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:42:36,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 39 msec 2024-11-23T15:42:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-23T15:42:36,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T15:42:36,625 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:36,632 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=239 (was 240), OpenFileDescriptor=447 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=375 (was 393), ProcessCount=11 (was 11), AvailableMemoryMB=7406 (was 7681) 2024-11-23T15:42:36,640 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=239, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=375, ProcessCount=11, AvailableMemoryMB=7405 2024-11-23T15:42:36,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T15:42:36,641 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:42:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T15:42:36,643 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T15:42:36,643 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:36,643 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 148 2024-11-23T15:42:36,643 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T15:42:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-23T15:42:36,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742331_1507 (size=959) 2024-11-23T15:42:36,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-23T15:42:36,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-23T15:42:37,053 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e 2024-11-23T15:42:37,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742332_1508 (size=53) 2024-11-23T15:42:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-23T15:42:37,451 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T15:42:37,461 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:42:37,461 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f77eb2e3d7d1fc2f4113a5c7f8d27655, disabling compactions & flushes 2024-11-23T15:42:37,461 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:37,461 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:37,461 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. after waiting 0 ms 2024-11-23T15:42:37,462 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:37,462 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:37,462 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: Waiting for close lock at 1732376557461Disabling compacts and flushes for region at 1732376557461Disabling writes for close at 1732376557462 (+1 ms)Writing region close event to WAL at 1732376557462Closed at 1732376557462 2024-11-23T15:42:37,464 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T15:42:37,465 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732376557464"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732376557464"}]},"ts":"1732376557464"} 2024-11-23T15:42:37,468 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T15:42:37,469 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T15:42:37,469 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376557469"}]},"ts":"1732376557469"} 2024-11-23T15:42:37,470 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T15:42:37,471 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f77eb2e3d7d1fc2f4113a5c7f8d27655, ASSIGN}] 2024-11-23T15:42:37,471 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f77eb2e3d7d1fc2f4113a5c7f8d27655, ASSIGN 2024-11-23T15:42:37,472 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f77eb2e3d7d1fc2f4113a5c7f8d27655, ASSIGN; state=OFFLINE, location=b712f9af2c12,46589,1732376409194; forceNewPlan=false, retain=false 2024-11-23T15:42:37,623 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=f77eb2e3d7d1fc2f4113a5c7f8d27655, regionState=OPENING, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:37,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f77eb2e3d7d1fc2f4113a5c7f8d27655, ASSIGN because future has completed 2024-11-23T15:42:37,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194}] 2024-11-23T15:42:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-23T15:42:37,785 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:37,785 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7752): Opening region: {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:42:37,785 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:37,786 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:42:37,786 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7794): checking encryption for f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:37,786 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7797): checking classloading for f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:37,788 INFO [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:37,790 INFO [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f77eb2e3d7d1fc2f4113a5c7f8d27655 columnFamilyName A 2024-11-23T15:42:37,790 DEBUG [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:37,791 INFO [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] regionserver.HStore(327): Store=f77eb2e3d7d1fc2f4113a5c7f8d27655/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:42:37,791 INFO [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:37,793 INFO [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f77eb2e3d7d1fc2f4113a5c7f8d27655 columnFamilyName B 2024-11-23T15:42:37,793 DEBUG [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:37,793 INFO [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] regionserver.HStore(327): Store=f77eb2e3d7d1fc2f4113a5c7f8d27655/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:42:37,794 INFO [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:37,795 INFO [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f77eb2e3d7d1fc2f4113a5c7f8d27655 columnFamilyName C 2024-11-23T15:42:37,796 DEBUG [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:37,796 INFO [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] regionserver.HStore(327): Store=f77eb2e3d7d1fc2f4113a5c7f8d27655/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:42:37,797 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1038): replaying wal for f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:37,798 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:37,798 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:37,800 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1048): stopping wal replay for f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:37,800 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1060): Cleaning up temporary data for f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:37,801 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T15:42:37,803 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1093): writing seq id for f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:37,806 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T15:42:37,807 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1114): Opened f77eb2e3d7d1fc2f4113a5c7f8d27655; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70129940, jitterRate=0.0450175404548645}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T15:42:37,807 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:37,808 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1006): Region open journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: Running coprocessor pre-open hook at 1732376557786Writing region info on filesystem at 1732376557786Initializing all the Stores at 1732376557787 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376557787Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376557787Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376557787Cleaning up temporary data from old regions at 1732376557800 (+13 ms)Running coprocessor post-open hooks at 1732376557807 (+7 ms)Region opened successfully at 1732376557808 (+1 ms) 2024-11-23T15:42:37,809 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., pid=150, masterSystemTime=1732376557782 2024-11-23T15:42:37,811 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:37,811 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:37,811 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=f77eb2e3d7d1fc2f4113a5c7f8d27655, regionState=OPEN, openSeqNum=2, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:37,813 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 because future has completed 2024-11-23T15:42:37,815 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-23T15:42:37,815 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 in 186 msec 2024-11-23T15:42:37,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-23T15:42:37,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f77eb2e3d7d1fc2f4113a5c7f8d27655, ASSIGN in 345 msec 2024-11-23T15:42:37,818 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T15:42:37,818 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376557818"}]},"ts":"1732376557818"} 2024-11-23T15:42:37,820 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T15:42:37,821 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=148, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T15:42:37,823 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1800 sec 2024-11-23T15:42:38,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-23T15:42:38,795 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:38,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a04de68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:38,796 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:42:38,796 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:42:38,797 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:42:38,798 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:42:38,798 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:42:38,798 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a9f803e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:38,798 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:42:38,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:42:38,799 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:38,800 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:42:38,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd8f3a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:38,802 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:42:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:42:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:42:38,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:42:38,806 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T15:42:38,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is b712f9af2c12,42351,1732376408377 2024-11-23T15:42:38,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2f2bf67a 2024-11-23T15:42:38,806 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T15:42:38,807 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T15:42:38,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T15:42:38,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T15:42:38,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-23T15:42:38,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742333_1509 (size=995) 2024-11-23T15:42:39,223 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-23T15:42:39,223 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-23T15:42:39,228 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T15:42:39,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f77eb2e3d7d1fc2f4113a5c7f8d27655, REOPEN/MOVE}] 2024-11-23T15:42:39,232 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f77eb2e3d7d1fc2f4113a5c7f8d27655, REOPEN/MOVE 2024-11-23T15:42:39,233 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=f77eb2e3d7d1fc2f4113a5c7f8d27655, regionState=CLOSING, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:39,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f77eb2e3d7d1fc2f4113a5c7f8d27655, REOPEN/MOVE because future has completed 2024-11-23T15:42:39,234 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T15:42:39,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194}] 2024-11-23T15:42:39,387 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(122): Close f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:39,387 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T15:42:39,387 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1722): Closing f77eb2e3d7d1fc2f4113a5c7f8d27655, disabling compactions & flushes 2024-11-23T15:42:39,387 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:39,387 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:39,387 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. after waiting 0 ms 2024-11-23T15:42:39,387 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:39,390 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-23T15:42:39,391 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:39,391 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegion(1676): Region close journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: Waiting for close lock at 1732376559387Running coprocessor pre-close hooks at 1732376559387Disabling compacts and flushes for region at 1732376559387Disabling writes for close at 1732376559387Writing region close event to WAL at 1732376559388 (+1 ms)Running coprocessor post-close hooks at 1732376559391 (+3 ms)Closed at 1732376559391 2024-11-23T15:42:39,391 WARN [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] regionserver.HRegionServer(3299): Not adding moved region record: f77eb2e3d7d1fc2f4113a5c7f8d27655 to self. 2024-11-23T15:42:39,393 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=154}] handler.UnassignRegionHandler(157): Closed f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:39,394 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=f77eb2e3d7d1fc2f4113a5c7f8d27655, regionState=CLOSED 2024-11-23T15:42:39,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; CloseRegionProcedure f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 because future has completed 2024-11-23T15:42:39,399 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-23T15:42:39,399 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; CloseRegionProcedure f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 in 163 msec 2024-11-23T15:42:39,400 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f77eb2e3d7d1fc2f4113a5c7f8d27655, REOPEN/MOVE; state=CLOSED, location=b712f9af2c12,46589,1732376409194; forceNewPlan=false, retain=true 2024-11-23T15:42:39,551 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=f77eb2e3d7d1fc2f4113a5c7f8d27655, regionState=OPENING, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:39,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f77eb2e3d7d1fc2f4113a5c7f8d27655, REOPEN/MOVE because future has completed 2024-11-23T15:42:39,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194}] 2024-11-23T15:42:39,714 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:39,715 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7752): Opening region: {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} 2024-11-23T15:42:39,715 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:39,715 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T15:42:39,715 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7794): checking encryption for f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:39,715 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(7797): checking classloading for f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:39,717 INFO [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:39,718 INFO [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f77eb2e3d7d1fc2f4113a5c7f8d27655 columnFamilyName A 2024-11-23T15:42:39,719 DEBUG [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:39,720 INFO [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] regionserver.HStore(327): Store=f77eb2e3d7d1fc2f4113a5c7f8d27655/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:42:39,720 INFO [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:39,721 INFO [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f77eb2e3d7d1fc2f4113a5c7f8d27655 columnFamilyName B 2024-11-23T15:42:39,721 DEBUG [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:39,722 INFO [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] regionserver.HStore(327): Store=f77eb2e3d7d1fc2f4113a5c7f8d27655/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:42:39,722 INFO [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:39,723 INFO [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f77eb2e3d7d1fc2f4113a5c7f8d27655 columnFamilyName C 2024-11-23T15:42:39,723 DEBUG [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:39,724 INFO [StoreOpener-f77eb2e3d7d1fc2f4113a5c7f8d27655-1 {}] regionserver.HStore(327): Store=f77eb2e3d7d1fc2f4113a5c7f8d27655/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T15:42:39,724 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1038): replaying wal for f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:39,725 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:39,726 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:39,727 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1048): stopping wal replay for f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:39,727 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1060): Cleaning up temporary data for f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:39,728 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T15:42:39,729 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1093): writing seq id for f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:39,730 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1114): Opened f77eb2e3d7d1fc2f4113a5c7f8d27655; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68206378, jitterRate=0.016354233026504517}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T15:42:39,730 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:39,731 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegion(1006): Region open journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: Running coprocessor pre-open hook at 1732376559715Writing region info on filesystem at 1732376559715Initializing all the Stores at 1732376559716 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376559716Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376559717 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732376559717Cleaning up temporary data from old regions at 1732376559727 (+10 ms)Running coprocessor post-open hooks at 1732376559730 (+3 ms)Region opened successfully at 1732376559731 (+1 ms) 2024-11-23T15:42:39,732 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., pid=155, masterSystemTime=1732376559711 2024-11-23T15:42:39,734 DEBUG [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:39,734 INFO [RS_OPEN_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_OPEN_REGION, pid=155}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:39,734 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=f77eb2e3d7d1fc2f4113a5c7f8d27655, regionState=OPEN, openSeqNum=5, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:39,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 because future has completed 2024-11-23T15:42:39,739 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=153 2024-11-23T15:42:39,739 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 in 181 msec 2024-11-23T15:42:39,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-23T15:42:39,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f77eb2e3d7d1fc2f4113a5c7f8d27655, REOPEN/MOVE in 508 msec 2024-11-23T15:42:39,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-23T15:42:39,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 515 msec 2024-11-23T15:42:39,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 937 msec 2024-11-23T15:42:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-23T15:42:39,750 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:39,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:42:39,751 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:42:39,752 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:42:39,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:42:39,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:42:39,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@641f62b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:42:39,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:42:39,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:39,754 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:42:39,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15e111a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:42:39,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:42:39,756 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:42:39,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:42:39,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:42:39,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a18bb3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:42:39,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:42:39,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:39,758 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:42:39,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7591847e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,759 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:42:39,759 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:42:39,760 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:42:39,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:42:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:42:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5236329d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:42:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:42:39,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:39,762 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:42:39,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6986f18a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,764 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:42:39,764 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:42:39,765 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:42:39,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:42:39,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:42:39,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39dc6e8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:42:39,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:42:39,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:39,767 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:42:39,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d1d26ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,768 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:42:39,768 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:42:39,769 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:42:39,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:42:39,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:42:39,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11627c1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:42:39,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:42:39,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:39,771 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53390, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:42:39,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a85e3e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,773 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,773 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:42:39,773 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:42:39,773 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:42:39,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:42:39,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:42:39,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c884fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:42:39,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:42:39,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:39,775 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53402, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:42:39,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24b67016, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,777 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:42:39,777 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:42:39,778 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:42:39,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:42:39,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:42:39,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4745b9a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:42:39,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:42:39,778 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:39,779 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:42:39,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1612ee16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,781 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:42:39,781 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:42:39,782 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:42:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:42:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:42:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48e93f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:42:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:42:39,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:39,783 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53438, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:42:39,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@376314f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,785 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:42:39,785 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:42:39,785 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:42:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:42:39,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:42:39,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2480fdc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:42:39,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:42:39,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:39,786 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:42:39,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@280f50c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,787 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,787 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b712f9af2c12,42351,-1 for getting cluster id 2024-11-23T15:42:39,787 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T15:42:39,788 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193' 2024-11-23T15:42:39,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T15:42:39,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f4f0ab8a-c15d-4d8b-8228-d8e9dbe50193" 2024-11-23T15:42:39,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bcf0eb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b712f9af2c12,42351,-1] 2024-11-23T15:42:39,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T15:42:39,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:39,789 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T15:42:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ce22f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T15:42:39,790 DEBUG [Thread-2295 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:42:39,790 DEBUG [Thread-2299 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:42:39,790 DEBUG [Thread-2293 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:42:39,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:42:39,790 DEBUG [Thread-2284 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:42:39,790 DEBUG [Thread-2288 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:42:39,791 DEBUG [Thread-2280 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:42:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:42:39,791 DEBUG [Thread-2299 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:42:39,791 DEBUG [Thread-2282 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:42:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:42:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:42:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:42:39,791 DEBUG [Thread-2291 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:42:39,791 DEBUG [Thread-2299 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:42:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:42:39,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:42:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-23T15:42:39,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:42:39,792 DEBUG [Thread-2297 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:42:39,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:42:39,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:42:39,792 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:42:39,792 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:42:39,792 DEBUG [Thread-2286 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T15:42:39,792 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:42:39,792 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:42:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-23T15:42:39,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:42:39,793 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:42:39,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:42:39,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:42:39,793 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:42:39,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:42:39,793 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:42:39,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:42:39,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:42:39,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b712f9af2c12,46589,1732376409194, seqNum=-1] 2024-11-23T15:42:39,794 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:42:39,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T15:42:39,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:42:39,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:42:39,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:42:39,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52782, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:42:39,794 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:42:39,795 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T15:42:39,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:42:39,796 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:42:39,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:42:39,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:42:39,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:42:39,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:42:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:42:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:42:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:42:39,799 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5] 2024-11-23T15:42:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:39,802 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:39,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T15:42:39,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376619824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:39,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:39,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:39,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376619827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:39,828 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:39,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:39,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:39,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:39,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112352bebd9b321f49588d77b92a03dace20_f77eb2e3d7d1fc2f4113a5c7f8d27655 is 50, key is test_row_0/A:col10/1732376559802/Put/seqid=0 2024-11-23T15:42:39,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376619828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:39,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376619828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:39,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376619828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:39,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:39,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742334_1510 (size=12154) 2024-11-23T15:42:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-23T15:42:39,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376619934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:39,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376619934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:39,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:39,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:39,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376619934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:39,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:39,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:39,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:39,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376619944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:39,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376619944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:39,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-23T15:42:39,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:39,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:39,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:39,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:39,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:40,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-23T15:42:40,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:40,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:40,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:40,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:40,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-23T15:42:40,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376620144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:40,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:40,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376620144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:40,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,146 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:40,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:40,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376620144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:40,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:40,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376620154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,156 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:40,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376620154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:40,252 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:40,255 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112352bebd9b321f49588d77b92a03dace20_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112352bebd9b321f49588d77b92a03dace20_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:40,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-23T15:42:40,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:40,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:40,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:40,256 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/4dd962ba33ac42439fa1635f9ad2594a, store: [table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:40,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:40,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:40,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/4dd962ba33ac42439fa1635f9ad2594a is 175, key is test_row_0/A:col10/1732376559802/Put/seqid=0 2024-11-23T15:42:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:40,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742335_1511 (size=30955) 2024-11-23T15:42:40,259 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/4dd962ba33ac42439fa1635f9ad2594a 2024-11-23T15:42:40,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/e82f676f867a479aa5f253239be6df49 is 50, key is test_row_0/B:col10/1732376559802/Put/seqid=0 2024-11-23T15:42:40,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742336_1512 (size=12001) 2024-11-23T15:42:40,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-23T15:42:40,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:40,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:40,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:40,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:40,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:40,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:40,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-23T15:42:40,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:40,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376620454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:40,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:40,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376620454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:40,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:40,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:40,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376620454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:40,456 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:40,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:40,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:40,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376620464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:40,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:40,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376620464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:40,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:40,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:40,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-23T15:42:40,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:40,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:40,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:40,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:40,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:40,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:40,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/e82f676f867a479aa5f253239be6df49 2024-11-23T15:42:40,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/2dd734aa745441c780edd3eea9b76b91 is 50, key is test_row_0/C:col10/1732376559802/Put/seqid=0 2024-11-23T15:42:40,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-23T15:42:40,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742337_1513 (size=12001) 2024-11-23T15:42:40,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:40,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:40,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:40,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:40,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:40,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-23T15:42:40,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:40,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:40,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:40,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:40,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:40,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-23T15:42:40,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:40,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376620964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:40,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376620964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:40,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376620964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:40,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:40,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376620974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:40,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376620975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:41,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-23T15:42:41,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:41,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:41,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:41,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:41,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:41,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:41,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/2dd734aa745441c780edd3eea9b76b91 2024-11-23T15:42:41,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/4dd962ba33ac42439fa1635f9ad2594a as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/4dd962ba33ac42439fa1635f9ad2594a 2024-11-23T15:42:41,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/4dd962ba33ac42439fa1635f9ad2594a, entries=150, sequenceid=16, filesize=30.2 K 2024-11-23T15:42:41,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/e82f676f867a479aa5f253239be6df49 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/e82f676f867a479aa5f253239be6df49 2024-11-23T15:42:41,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/e82f676f867a479aa5f253239be6df49, entries=150, sequenceid=16, filesize=11.7 K 2024-11-23T15:42:41,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/2dd734aa745441c780edd3eea9b76b91 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2dd734aa745441c780edd3eea9b76b91 2024-11-23T15:42:41,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2dd734aa745441c780edd3eea9b76b91, entries=150, sequenceid=16, filesize=11.7 K 2024-11-23T15:42:41,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for f77eb2e3d7d1fc2f4113a5c7f8d27655 in 1328ms, sequenceid=16, compaction requested=false 2024-11-23T15:42:41,130 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-23T15:42:41,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:41,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-23T15:42:41,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:41,176 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T15:42:41,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112380d9346a18f4455080da5d7b0216d275_f77eb2e3d7d1fc2f4113a5c7f8d27655 is 50, key is test_row_0/A:col10/1732376559823/Put/seqid=0 2024-11-23T15:42:41,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742338_1514 (size=12154) 2024-11-23T15:42:41,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,202 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112380d9346a18f4455080da5d7b0216d275_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112380d9346a18f4455080da5d7b0216d275_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:41,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/4e2f7ca2de3545bb9ff6aa735449cda2, store: [table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:41,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/4e2f7ca2de3545bb9ff6aa735449cda2 is 175, key is test_row_0/A:col10/1732376559823/Put/seqid=0 2024-11-23T15:42:41,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742339_1515 (size=30955) 2024-11-23T15:42:41,211 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/4e2f7ca2de3545bb9ff6aa735449cda2 2024-11-23T15:42:41,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/35402dbc187b45f9816913fc33e3e5ea is 50, key is test_row_0/B:col10/1732376559823/Put/seqid=0 2024-11-23T15:42:41,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742340_1516 (size=12001) 2024-11-23T15:42:41,232 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/35402dbc187b45f9816913fc33e3e5ea 2024-11-23T15:42:41,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/9211eb22c49447d5bf17a0cd0c1a8350 is 50, key is test_row_0/C:col10/1732376559823/Put/seqid=0 2024-11-23T15:42:41,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742341_1517 (size=12001) 2024-11-23T15:42:41,264 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/9211eb22c49447d5bf17a0cd0c1a8350 2024-11-23T15:42:41,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/4e2f7ca2de3545bb9ff6aa735449cda2 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/4e2f7ca2de3545bb9ff6aa735449cda2 2024-11-23T15:42:41,273 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/4e2f7ca2de3545bb9ff6aa735449cda2, entries=150, sequenceid=41, filesize=30.2 K 2024-11-23T15:42:41,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/35402dbc187b45f9816913fc33e3e5ea as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/35402dbc187b45f9816913fc33e3e5ea 2024-11-23T15:42:41,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,277 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/35402dbc187b45f9816913fc33e3e5ea, entries=150, sequenceid=41, filesize=11.7 K 2024-11-23T15:42:41,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/9211eb22c49447d5bf17a0cd0c1a8350 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/9211eb22c49447d5bf17a0cd0c1a8350 2024-11-23T15:42:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,281 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/9211eb22c49447d5bf17a0cd0c1a8350, entries=150, sequenceid=41, filesize=11.7 K 2024-11-23T15:42:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,282 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for f77eb2e3d7d1fc2f4113a5c7f8d27655 in 106ms, sequenceid=41, compaction requested=false 2024-11-23T15:42:41,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:41,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:41,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-23T15:42:41,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-23T15:42:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-23T15:42:41,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4870 sec 2024-11-23T15:42:41,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,287 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.4960 sec 2024-11-23T15:42:41,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,399 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T15:42:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-23T15:42:41,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,944 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:41,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,944 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:42:41,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-23T15:42:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-23T15:42:41,947 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:42:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,947 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:42:41,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:42:41,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,018 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:42,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T15:42:42,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123a92aad3f198f41afa6280c99072019dc_f77eb2e3d7d1fc2f4113a5c7f8d27655 is 50, key is test_row_0/A:col10/1732376562018/Put/seqid=0 2024-11-23T15:42:42,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742343_1519 (size=26798) 2024-11-23T15:42:42,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-23T15:42:42,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376622051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:42,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:42,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376622052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,057 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:42,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376622053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:42,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376622053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:42,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:42,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376622054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:42,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:42,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:42,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-23T15:42:42,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:42,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:42,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:42,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:42,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:42,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376622164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376622164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376622164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376622164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:42,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:42,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376622164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:42,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:42,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-23T15:42:42,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:42,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:42,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:42,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:42,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-23T15:42:42,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376622374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:42,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376622374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376622374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:42,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:42,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376622374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:42,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376622374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:42,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:42,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-23T15:42:42,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:42,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:42,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:42,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:42,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:42,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:42,445 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:42,448 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123a92aad3f198f41afa6280c99072019dc_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123a92aad3f198f41afa6280c99072019dc_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:42,449 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/688199b0e1114e879cd3babc2377c316, store: [table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:42,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/688199b0e1114e879cd3babc2377c316 is 175, key is test_row_0/A:col10/1732376562018/Put/seqid=0 2024-11-23T15:42:42,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742342_1518 (size=82585) 2024-11-23T15:42:42,451 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/688199b0e1114e879cd3babc2377c316 2024-11-23T15:42:42,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/05e46f9a2943490387d88644c49db8db is 50, key is test_row_0/B:col10/1732376562018/Put/seqid=0 2024-11-23T15:42:42,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742344_1520 (size=12001) 2024-11-23T15:42:42,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-23T15:42:42,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:42,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:42,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:42,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:42,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:42,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-23T15:42:42,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376622684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:42,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376622685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:42,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,685 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:42,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:42,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376622685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:42,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:42,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:42,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376622685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:42,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:42,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:42,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376622685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:42,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:42,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:42,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-23T15:42:42,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:42,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:42,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:42,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:42,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:42,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-23T15:42:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:42,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:42,865 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/05e46f9a2943490387d88644c49db8db 2024-11-23T15:42:42,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/0460740ae86b426b98c98ca90a323239 is 50, key is test_row_0/C:col10/1732376562018/Put/seqid=0 2024-11-23T15:42:42,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742345_1521 (size=12001) 2024-11-23T15:42:42,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/0460740ae86b426b98c98ca90a323239 2024-11-23T15:42:42,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/688199b0e1114e879cd3babc2377c316 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/688199b0e1114e879cd3babc2377c316 2024-11-23T15:42:42,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/688199b0e1114e879cd3babc2377c316, entries=450, sequenceid=53, filesize=80.6 K 2024-11-23T15:42:42,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/05e46f9a2943490387d88644c49db8db as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/05e46f9a2943490387d88644c49db8db 2024-11-23T15:42:42,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/05e46f9a2943490387d88644c49db8db, entries=150, sequenceid=53, filesize=11.7 K 2024-11-23T15:42:42,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/0460740ae86b426b98c98ca90a323239 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/0460740ae86b426b98c98ca90a323239 2024-11-23T15:42:42,942 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/0460740ae86b426b98c98ca90a323239, entries=150, sequenceid=53, filesize=11.7 K 2024-11-23T15:42:42,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f77eb2e3d7d1fc2f4113a5c7f8d27655 in 925ms, sequenceid=53, compaction requested=true 2024-11-23T15:42:42,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:42,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f77eb2e3d7d1fc2f4113a5c7f8d27655:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:42:42,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:42,943 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:42,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f77eb2e3d7d1fc2f4113a5c7f8d27655:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:42:42,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:42,943 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:42,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f77eb2e3d7d1fc2f4113a5c7f8d27655:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:42:42,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:42,944 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 144495 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:42,944 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:42,944 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): f77eb2e3d7d1fc2f4113a5c7f8d27655/B is initiating minor compaction (all files) 2024-11-23T15:42:42,944 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): f77eb2e3d7d1fc2f4113a5c7f8d27655/A is initiating minor compaction (all files) 2024-11-23T15:42:42,945 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f77eb2e3d7d1fc2f4113a5c7f8d27655/B in TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:42,945 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/e82f676f867a479aa5f253239be6df49, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/35402dbc187b45f9816913fc33e3e5ea, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/05e46f9a2943490387d88644c49db8db] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp, totalSize=35.2 K 2024-11-23T15:42:42,945 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f77eb2e3d7d1fc2f4113a5c7f8d27655/A in TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:42,945 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/4dd962ba33ac42439fa1635f9ad2594a, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/4e2f7ca2de3545bb9ff6aa735449cda2, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/688199b0e1114e879cd3babc2377c316] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp, totalSize=141.1 K 2024-11-23T15:42:42,945 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:42,945 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. files: [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/4dd962ba33ac42439fa1635f9ad2594a, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/4e2f7ca2de3545bb9ff6aa735449cda2, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/688199b0e1114e879cd3babc2377c316] 2024-11-23T15:42:42,945 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting e82f676f867a479aa5f253239be6df49, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732376559801 2024-11-23T15:42:42,946 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dd962ba33ac42439fa1635f9ad2594a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732376559801 2024-11-23T15:42:42,946 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 35402dbc187b45f9816913fc33e3e5ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732376559823 2024-11-23T15:42:42,946 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e2f7ca2de3545bb9ff6aa735449cda2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732376559823 2024-11-23T15:42:42,946 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 05e46f9a2943490387d88644c49db8db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732376562010 2024-11-23T15:42:42,946 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 688199b0e1114e879cd3babc2377c316, keycount=450, bloomtype=ROW, size=80.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732376561994 2024-11-23T15:42:42,956 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f77eb2e3d7d1fc2f4113a5c7f8d27655#B#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:42,957 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/971fa387c67c423481ea6030ed11aa46 is 50, key is test_row_0/B:col10/1732376562018/Put/seqid=0 2024-11-23T15:42:42,967 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:42,975 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411233d4c2601986042fb82610990763ad618_f77eb2e3d7d1fc2f4113a5c7f8d27655 store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:42,978 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411233d4c2601986042fb82610990763ad618_f77eb2e3d7d1fc2f4113a5c7f8d27655, store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:42,978 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411233d4c2601986042fb82610990763ad618_f77eb2e3d7d1fc2f4113a5c7f8d27655 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:42,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742346_1522 (size=12104) 2024-11-23T15:42:42,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742347_1523 (size=4469) 2024-11-23T15:42:42,987 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f77eb2e3d7d1fc2f4113a5c7f8d27655#A#compaction#451 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:42,988 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/974c6f4b4a6c4f92a8a818dc020b2e75 is 175, key is test_row_0/A:col10/1732376562018/Put/seqid=0 2024-11-23T15:42:42,991 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/971fa387c67c423481ea6030ed11aa46 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/971fa387c67c423481ea6030ed11aa46 2024-11-23T15:42:42,996 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f77eb2e3d7d1fc2f4113a5c7f8d27655/B of f77eb2e3d7d1fc2f4113a5c7f8d27655 into 971fa387c67c423481ea6030ed11aa46(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:42,996 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:42,996 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., storeName=f77eb2e3d7d1fc2f4113a5c7f8d27655/B, priority=13, startTime=1732376562943; duration=0sec 2024-11-23T15:42:42,997 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:42,997 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f77eb2e3d7d1fc2f4113a5c7f8d27655:B 2024-11-23T15:42:42,997 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:42,998 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:42,998 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): f77eb2e3d7d1fc2f4113a5c7f8d27655/C is initiating minor compaction (all files) 2024-11-23T15:42:42,998 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f77eb2e3d7d1fc2f4113a5c7f8d27655/C in TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:42,999 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2dd734aa745441c780edd3eea9b76b91, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/9211eb22c49447d5bf17a0cd0c1a8350, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/0460740ae86b426b98c98ca90a323239] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp, totalSize=35.2 K 2024-11-23T15:42:42,999 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dd734aa745441c780edd3eea9b76b91, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732376559801 2024-11-23T15:42:42,999 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 9211eb22c49447d5bf17a0cd0c1a8350, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732376559823 2024-11-23T15:42:42,999 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 0460740ae86b426b98c98ca90a323239, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732376562010 2024-11-23T15:42:43,007 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f77eb2e3d7d1fc2f4113a5c7f8d27655#C#compaction#452 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:43,008 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/126cfb273c5a45ec8865819d8a1e4908 is 50, key is test_row_0/C:col10/1732376562018/Put/seqid=0 2024-11-23T15:42:43,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742348_1524 (size=31058) 2024-11-23T15:42:43,043 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/974c6f4b4a6c4f92a8a818dc020b2e75 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/974c6f4b4a6c4f92a8a818dc020b2e75 2024-11-23T15:42:43,048 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f77eb2e3d7d1fc2f4113a5c7f8d27655/A of f77eb2e3d7d1fc2f4113a5c7f8d27655 into 974c6f4b4a6c4f92a8a818dc020b2e75(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:43,048 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:43,048 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., storeName=f77eb2e3d7d1fc2f4113a5c7f8d27655/A, priority=13, startTime=1732376562943; duration=0sec 2024-11-23T15:42:43,049 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:43,049 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f77eb2e3d7d1fc2f4113a5c7f8d27655:A 2024-11-23T15:42:43,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-23T15:42:43,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:43,051 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T15:42:43,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742349_1525 (size=12104) 2024-11-23T15:42:43,062 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/126cfb273c5a45ec8865819d8a1e4908 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/126cfb273c5a45ec8865819d8a1e4908 2024-11-23T15:42:43,072 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f77eb2e3d7d1fc2f4113a5c7f8d27655/C of f77eb2e3d7d1fc2f4113a5c7f8d27655 into 126cfb273c5a45ec8865819d8a1e4908(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:43,072 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:43,072 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., storeName=f77eb2e3d7d1fc2f4113a5c7f8d27655/C, priority=13, startTime=1732376562943; duration=0sec 2024-11-23T15:42:43,072 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:43,072 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f77eb2e3d7d1fc2f4113a5c7f8d27655:C 2024-11-23T15:42:43,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411230d8a0c1155b440708712674a40d3a718_f77eb2e3d7d1fc2f4113a5c7f8d27655 is 50, key is test_row_0/A:col10/1732376562052/Put/seqid=0 2024-11-23T15:42:43,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-23T15:42:43,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742350_1526 (size=12154) 2024-11-23T15:42:43,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:43,121 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411230d8a0c1155b440708712674a40d3a718_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230d8a0c1155b440708712674a40d3a718_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:43,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/d8ee4600e5624218bf8aa45c1be4ed99, store: [table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:43,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/d8ee4600e5624218bf8aa45c1be4ed99 is 175, key is test_row_0/A:col10/1732376562052/Put/seqid=0 2024-11-23T15:42:43,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742351_1527 (size=30955) 2024-11-23T15:42:43,127 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/d8ee4600e5624218bf8aa45c1be4ed99 2024-11-23T15:42:43,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/d722014b5c894364bf2a1052e148e749 is 50, key is test_row_0/B:col10/1732376562052/Put/seqid=0 2024-11-23T15:42:43,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742352_1528 (size=12001) 2024-11-23T15:42:43,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:43,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:43,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:43,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376623199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:43,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376623202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:43,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:43,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376623202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:43,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:43,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:43,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:43,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376623203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:43,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376623203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:43,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,206 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:43,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:43,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376623314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:43,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:43,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376623314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:43,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:43,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:43,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376623315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:43,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376623315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:43,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376623315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:43,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376623524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:43,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:43,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376623524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376623524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:43,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376623524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:43,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:43,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:43,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376623524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,527 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:43,556 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/d722014b5c894364bf2a1052e148e749 2024-11-23T15:42:43,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/47eb1413f0c3430da822489cc8f51cc6 is 50, key is test_row_0/C:col10/1732376562052/Put/seqid=0 2024-11-23T15:42:43,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742353_1529 (size=12001) 2024-11-23T15:42:43,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376623834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:43,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:43,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376623834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:43,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:43,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:43,836 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:43,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376623834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:43,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:43,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:43,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376623834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:43,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376623834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:43,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:43,973 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/47eb1413f0c3430da822489cc8f51cc6 2024-11-23T15:42:43,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/d8ee4600e5624218bf8aa45c1be4ed99 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/d8ee4600e5624218bf8aa45c1be4ed99 2024-11-23T15:42:43,979 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/d8ee4600e5624218bf8aa45c1be4ed99, entries=150, sequenceid=79, filesize=30.2 K 2024-11-23T15:42:43,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/d722014b5c894364bf2a1052e148e749 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/d722014b5c894364bf2a1052e148e749 2024-11-23T15:42:43,983 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/d722014b5c894364bf2a1052e148e749, entries=150, sequenceid=79, filesize=11.7 K 2024-11-23T15:42:43,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/47eb1413f0c3430da822489cc8f51cc6 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/47eb1413f0c3430da822489cc8f51cc6 2024-11-23T15:42:43,987 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/47eb1413f0c3430da822489cc8f51cc6, entries=150, sequenceid=79, filesize=11.7 K 2024-11-23T15:42:43,988 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for f77eb2e3d7d1fc2f4113a5c7f8d27655 in 937ms, sequenceid=79, compaction requested=false 2024-11-23T15:42:43,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:43,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:43,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-23T15:42:43,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-23T15:42:43,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-23T15:42:43,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0430 sec 2024-11-23T15:42:43,993 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.0480 sec 2024-11-23T15:42:44,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-23T15:42:44,104 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:44,104 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:42:44,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-23T15:42:44,106 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:42:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-23T15:42:44,106 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:42:44,107 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:42:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-23T15:42:44,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-23T15:42:44,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:44,259 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T15:42:44,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411231441f9ad06f442278fbe440171cc45c4_f77eb2e3d7d1fc2f4113a5c7f8d27655 is 50, key is test_row_0/A:col10/1732376563201/Put/seqid=0 2024-11-23T15:42:44,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742354_1530 (size=12154) 2024-11-23T15:42:44,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:44,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376624361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:44,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376624362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:44,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:44,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376624363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:44,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376624364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:44,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:44,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:44,368 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376624365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:44,368 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,368 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:44,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-23T15:42:44,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376624474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:44,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376624474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:44,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,476 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:44,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:44,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376624475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:44,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:44,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:44,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376624475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:44,476 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:44,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:44,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376624475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:44,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,477 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:44,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:44,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:44,675 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411231441f9ad06f442278fbe440171cc45c4_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231441f9ad06f442278fbe440171cc45c4_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:44,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/9564d9e22e9d471c85ee6338fb317bc9, store: [table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:44,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/9564d9e22e9d471c85ee6338fb317bc9 is 175, key is test_row_0/A:col10/1732376563201/Put/seqid=0 2024-11-23T15:42:44,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742355_1531 (size=30955) 2024-11-23T15:42:44,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376624683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:44,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376624684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:44,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376624684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:44,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376624684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376624684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:44,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-23T15:42:44,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:44,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376624994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:44,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:44,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:44,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376624994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:44,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376624995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:44,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:44,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:44,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:44,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:44,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376624995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:44,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:44,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376624995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:44,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:44,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:44,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:45,080 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/9564d9e22e9d471c85ee6338fb317bc9 2024-11-23T15:42:45,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/18b00f44939347f49efc1aae2b3264f7 is 50, key is test_row_0/B:col10/1732376563201/Put/seqid=0 2024-11-23T15:42:45,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742356_1532 (size=12001) 2024-11-23T15:42:45,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-23T15:42:45,489 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/18b00f44939347f49efc1aae2b3264f7 2024-11-23T15:42:45,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/f7e0ebf6b26342ad941ca911828427a3 is 50, key is test_row_0/C:col10/1732376563201/Put/seqid=0 2024-11-23T15:42:45,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742357_1533 (size=12001) 2024-11-23T15:42:45,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376625505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:45,508 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:45,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:45,508 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:45,508 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:45,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376625505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:45,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:45,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:45,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:45,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:45,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376625505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:45,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:45,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:45,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:45,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:45,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376625505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:45,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:45,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:45,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:45,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:45,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376625505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:45,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:45,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:45,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:45,900 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/f7e0ebf6b26342ad941ca911828427a3 2024-11-23T15:42:45,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/9564d9e22e9d471c85ee6338fb317bc9 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/9564d9e22e9d471c85ee6338fb317bc9 2024-11-23T15:42:45,906 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/9564d9e22e9d471c85ee6338fb317bc9, entries=150, sequenceid=92, filesize=30.2 K 2024-11-23T15:42:45,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/18b00f44939347f49efc1aae2b3264f7 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/18b00f44939347f49efc1aae2b3264f7 2024-11-23T15:42:45,910 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/18b00f44939347f49efc1aae2b3264f7, entries=150, sequenceid=92, filesize=11.7 K 2024-11-23T15:42:45,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/f7e0ebf6b26342ad941ca911828427a3 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f7e0ebf6b26342ad941ca911828427a3 2024-11-23T15:42:45,915 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f7e0ebf6b26342ad941ca911828427a3, entries=150, sequenceid=92, filesize=11.7 K 2024-11-23T15:42:45,915 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f77eb2e3d7d1fc2f4113a5c7f8d27655 in 1657ms, sequenceid=92, compaction requested=true 2024-11-23T15:42:45,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:45,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:45,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-23T15:42:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-23T15:42:45,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-23T15:42:45,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8120 sec 2024-11-23T15:42:45,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.8160 sec 2024-11-23T15:42:46,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-23T15:42:46,255 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:46,255 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:42:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-23T15:42:46,257 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:42:46,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-23T15:42:46,257 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:42:46,257 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:42:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-23T15:42:46,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-23T15:42:46,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:46,409 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T15:42:46,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123eede40c34c5149e0bc84ca2f22212386_f77eb2e3d7d1fc2f4113a5c7f8d27655 is 50, key is test_row_0/A:col10/1732376564363/Put/seqid=0 2024-11-23T15:42:46,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742358_1534 (size=12154) 2024-11-23T15:42:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:46,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:46,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376626525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:46,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376626527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:46,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:46,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376626528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:46,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:46,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:46,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376626529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:46,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:46,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376626530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,535 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-23T15:42:46,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376626634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:46,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376626634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:46,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:46,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376626634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:46,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376626644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:46,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376626644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:46,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,647 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:46,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:46,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:46,826 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123eede40c34c5149e0bc84ca2f22212386_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123eede40c34c5149e0bc84ca2f22212386_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:46,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/a32434d423cb454db827a722372c4ef7, store: [table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:46,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/a32434d423cb454db827a722372c4ef7 is 175, key is test_row_0/A:col10/1732376564363/Put/seqid=0 2024-11-23T15:42:46,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742359_1535 (size=30955) 2024-11-23T15:42:46,834 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/a32434d423cb454db827a722372c4ef7 2024-11-23T15:42:46,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/05ef5fc3384e463ea9df9bcd5aeacaa0 is 50, key is test_row_0/B:col10/1732376564363/Put/seqid=0 2024-11-23T15:42:46,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742360_1536 (size=12001) 2024-11-23T15:42:46,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376626844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376626844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376626844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:46,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:46,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376626854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:46,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:46,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376626854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:46,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-23T15:42:47,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:47,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376627154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:47,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:47,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376627154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:47,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:47,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:47,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376627154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:47,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:47,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376627164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:47,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:47,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376627165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,167 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:47,242 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/05ef5fc3384e463ea9df9bcd5aeacaa0 2024-11-23T15:42:47,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/cacf24dafd584064bbcf3cfdff328452 is 50, key is test_row_0/C:col10/1732376564363/Put/seqid=0 2024-11-23T15:42:47,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742361_1537 (size=12001) 2024-11-23T15:42:47,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-23T15:42:47,652 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/cacf24dafd584064bbcf3cfdff328452 2024-11-23T15:42:47,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/a32434d423cb454db827a722372c4ef7 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/a32434d423cb454db827a722372c4ef7 2024-11-23T15:42:47,658 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/a32434d423cb454db827a722372c4ef7, entries=150, sequenceid=116, filesize=30.2 K 2024-11-23T15:42:47,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/05ef5fc3384e463ea9df9bcd5aeacaa0 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/05ef5fc3384e463ea9df9bcd5aeacaa0 2024-11-23T15:42:47,662 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/05ef5fc3384e463ea9df9bcd5aeacaa0, entries=150, sequenceid=116, filesize=11.7 K 2024-11-23T15:42:47,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/cacf24dafd584064bbcf3cfdff328452 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/cacf24dafd584064bbcf3cfdff328452 2024-11-23T15:42:47,666 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/cacf24dafd584064bbcf3cfdff328452, entries=150, sequenceid=116, filesize=11.7 K 2024-11-23T15:42:47,667 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for f77eb2e3d7d1fc2f4113a5c7f8d27655 in 1258ms, sequenceid=116, compaction requested=true 2024-11-23T15:42:47,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:47,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:47,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-23T15:42:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-23T15:42:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:47,669 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:47,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T15:42:47,673 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-23T15:42:47,673 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4140 sec 2024-11-23T15:42:47,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.4180 sec 2024-11-23T15:42:47,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123577a72a415d64ec1aa946abfe3337d8a_f77eb2e3d7d1fc2f4113a5c7f8d27655 is 50, key is test_row_0/A:col10/1732376566529/Put/seqid=0 2024-11-23T15:42:47,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742362_1538 (size=14644) 2024-11-23T15:42:47,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376627692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376627693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:47,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:47,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376627694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:47,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376627695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:47,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376627696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:47,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376627804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:47,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376627804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376627804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:47,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376627804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:47,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376627804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:48,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376628014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:48,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:48,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376628014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:48,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:48,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376628014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:48,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:48,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376628014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:48,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376628014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:48,096 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:48,099 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123577a72a415d64ec1aa946abfe3337d8a_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123577a72a415d64ec1aa946abfe3337d8a_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:48,099 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/b7b09bec34074f00b6843f2e3f49e260, store: [table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:48,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/b7b09bec34074f00b6843f2e3f49e260 is 175, key is test_row_0/A:col10/1732376566529/Put/seqid=0 2024-11-23T15:42:48,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742363_1539 (size=39599) 2024-11-23T15:42:48,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376628324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:48,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376628324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:48,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376628324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:48,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376628324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:48,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376628324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:48,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-23T15:42:48,404 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:48,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:42:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-23T15:42:48,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-23T15:42:48,406 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:42:48,406 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:42:48,407 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:42:48,503 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/b7b09bec34074f00b6843f2e3f49e260 2024-11-23T15:42:48,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/4a21121ef4044b398883e94323f46c7b is 50, key is test_row_0/B:col10/1732376566529/Put/seqid=0 2024-11-23T15:42:48,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742364_1540 (size=12051) 2024-11-23T15:42:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-23T15:42:48,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-23T15:42:48,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:48,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:48,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:48,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:48,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:48,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-23T15:42:48,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:48,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:48,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:48,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:48,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:48,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:48,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-23T15:42:48,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:48,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376628834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:48,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:48,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376628834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:48,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:48,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376628835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:48,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:48,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376628835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:48,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:48,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:48,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376628844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:48,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:48,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:48,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-23T15:42:48,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:48,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:48,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:48,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:48,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:48,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:48,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/4a21121ef4044b398883e94323f46c7b 2024-11-23T15:42:48,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/2c9f74938975405f86ba8aa9bea470bf is 50, key is test_row_0/C:col10/1732376566529/Put/seqid=0 2024-11-23T15:42:48,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742365_1541 (size=12051) 2024-11-23T15:42:49,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-23T15:42:49,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:49,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:49,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:49,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:49,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:49,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-23T15:42:49,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-23T15:42:49,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:49,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:49,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:49,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:49,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:49,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:49,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/2c9f74938975405f86ba8aa9bea470bf 2024-11-23T15:42:49,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/b7b09bec34074f00b6843f2e3f49e260 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/b7b09bec34074f00b6843f2e3f49e260 2024-11-23T15:42:49,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/b7b09bec34074f00b6843f2e3f49e260, entries=200, sequenceid=130, filesize=38.7 K 2024-11-23T15:42:49,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-23T15:42:49,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/4a21121ef4044b398883e94323f46c7b as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/4a21121ef4044b398883e94323f46c7b 2024-11-23T15:42:49,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:49,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:49,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:49,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:49,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:49,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:49,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/4a21121ef4044b398883e94323f46c7b, entries=150, sequenceid=130, filesize=11.8 K 2024-11-23T15:42:49,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/2c9f74938975405f86ba8aa9bea470bf as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2c9f74938975405f86ba8aa9bea470bf 2024-11-23T15:42:49,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2c9f74938975405f86ba8aa9bea470bf, entries=150, sequenceid=130, filesize=11.8 K 2024-11-23T15:42:49,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for f77eb2e3d7d1fc2f4113a5c7f8d27655 in 1665ms, sequenceid=130, compaction requested=true 2024-11-23T15:42:49,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:49,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f77eb2e3d7d1fc2f4113a5c7f8d27655:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:42:49,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:49,334 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-23T15:42:49,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f77eb2e3d7d1fc2f4113a5c7f8d27655:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:42:49,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:49,334 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-23T15:42:49,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f77eb2e3d7d1fc2f4113a5c7f8d27655:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:42:49,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:49,335 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 163522 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-23T15:42:49,335 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60158 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-23T15:42:49,335 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): f77eb2e3d7d1fc2f4113a5c7f8d27655/B is initiating minor compaction (all files) 2024-11-23T15:42:49,335 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): f77eb2e3d7d1fc2f4113a5c7f8d27655/A is initiating minor compaction (all files) 2024-11-23T15:42:49,335 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f77eb2e3d7d1fc2f4113a5c7f8d27655/A in TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:49,335 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f77eb2e3d7d1fc2f4113a5c7f8d27655/B in TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:49,335 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/971fa387c67c423481ea6030ed11aa46, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/d722014b5c894364bf2a1052e148e749, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/18b00f44939347f49efc1aae2b3264f7, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/05ef5fc3384e463ea9df9bcd5aeacaa0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/4a21121ef4044b398883e94323f46c7b] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp, totalSize=58.7 K 2024-11-23T15:42:49,335 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/974c6f4b4a6c4f92a8a818dc020b2e75, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/d8ee4600e5624218bf8aa45c1be4ed99, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/9564d9e22e9d471c85ee6338fb317bc9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/a32434d423cb454db827a722372c4ef7, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/b7b09bec34074f00b6843f2e3f49e260] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp, totalSize=159.7 K 2024-11-23T15:42:49,335 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:49,335 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. files: [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/974c6f4b4a6c4f92a8a818dc020b2e75, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/d8ee4600e5624218bf8aa45c1be4ed99, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/9564d9e22e9d471c85ee6338fb317bc9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/a32434d423cb454db827a722372c4ef7, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/b7b09bec34074f00b6843f2e3f49e260] 2024-11-23T15:42:49,336 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 971fa387c67c423481ea6030ed11aa46, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732376562010 2024-11-23T15:42:49,336 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 974c6f4b4a6c4f92a8a818dc020b2e75, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732376562010 2024-11-23T15:42:49,336 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting d722014b5c894364bf2a1052e148e749, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732376562049 2024-11-23T15:42:49,336 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8ee4600e5624218bf8aa45c1be4ed99, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732376562049 2024-11-23T15:42:49,336 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 18b00f44939347f49efc1aae2b3264f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732376563200 2024-11-23T15:42:49,336 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9564d9e22e9d471c85ee6338fb317bc9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732376563200 2024-11-23T15:42:49,336 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 05ef5fc3384e463ea9df9bcd5aeacaa0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732376564362 2024-11-23T15:42:49,336 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting a32434d423cb454db827a722372c4ef7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732376564362 2024-11-23T15:42:49,336 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a21121ef4044b398883e94323f46c7b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732376566516 2024-11-23T15:42:49,336 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7b09bec34074f00b6843f2e3f49e260, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732376566516 2024-11-23T15:42:49,341 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:49,342 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f77eb2e3d7d1fc2f4113a5c7f8d27655#B#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:49,342 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123576aaa25e9e8458e939fbf32141406bd_f77eb2e3d7d1fc2f4113a5c7f8d27655 store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:49,343 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/b0bbdc753d154b2fbc3e45aac4575851 is 50, key is test_row_0/B:col10/1732376566529/Put/seqid=0 2024-11-23T15:42:49,345 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123576aaa25e9e8458e939fbf32141406bd_f77eb2e3d7d1fc2f4113a5c7f8d27655, store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:49,345 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123576aaa25e9e8458e939fbf32141406bd_f77eb2e3d7d1fc2f4113a5c7f8d27655 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:49,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742366_1542 (size=12325) 2024-11-23T15:42:49,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742367_1543 (size=4469) 2024-11-23T15:42:49,357 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f77eb2e3d7d1fc2f4113a5c7f8d27655#A#compaction#466 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:49,357 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/1453e986da1b4f99bc5ee88d6cfbbdad is 175, key is test_row_0/A:col10/1732376566529/Put/seqid=0 2024-11-23T15:42:49,358 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/b0bbdc753d154b2fbc3e45aac4575851 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/b0bbdc753d154b2fbc3e45aac4575851 2024-11-23T15:42:49,361 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f77eb2e3d7d1fc2f4113a5c7f8d27655/B of f77eb2e3d7d1fc2f4113a5c7f8d27655 into b0bbdc753d154b2fbc3e45aac4575851(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:49,361 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:49,361 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., storeName=f77eb2e3d7d1fc2f4113a5c7f8d27655/B, priority=11, startTime=1732376569334; duration=0sec 2024-11-23T15:42:49,361 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:49,361 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f77eb2e3d7d1fc2f4113a5c7f8d27655:B 2024-11-23T15:42:49,361 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-23T15:42:49,362 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60158 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-23T15:42:49,362 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): f77eb2e3d7d1fc2f4113a5c7f8d27655/C is initiating minor compaction (all files) 2024-11-23T15:42:49,362 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f77eb2e3d7d1fc2f4113a5c7f8d27655/C in TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:49,362 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/126cfb273c5a45ec8865819d8a1e4908, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/47eb1413f0c3430da822489cc8f51cc6, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f7e0ebf6b26342ad941ca911828427a3, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/cacf24dafd584064bbcf3cfdff328452, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2c9f74938975405f86ba8aa9bea470bf] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp, totalSize=58.7 K 2024-11-23T15:42:49,363 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 126cfb273c5a45ec8865819d8a1e4908, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732376562010 2024-11-23T15:42:49,363 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 47eb1413f0c3430da822489cc8f51cc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732376562049 2024-11-23T15:42:49,363 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting f7e0ebf6b26342ad941ca911828427a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732376563200 2024-11-23T15:42:49,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742368_1544 (size=31279) 2024-11-23T15:42:49,364 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting cacf24dafd584064bbcf3cfdff328452, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732376564362 2024-11-23T15:42:49,364 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c9f74938975405f86ba8aa9bea470bf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732376566516 2024-11-23T15:42:49,371 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f77eb2e3d7d1fc2f4113a5c7f8d27655#C#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:49,372 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/65f8d0d38b464c5f8464edd7c4d8c8c8 is 50, key is test_row_0/C:col10/1732376566529/Put/seqid=0 2024-11-23T15:42:49,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742369_1545 (size=12325) 2024-11-23T15:42:49,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-23T15:42:49,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:49,480 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-23T15:42:49,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112307403055b35847ad86c9c0a99610f488_f77eb2e3d7d1fc2f4113a5c7f8d27655 is 50, key is test_row_0/A:col10/1732376567694/Put/seqid=0 2024-11-23T15:42:49,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742370_1546 (size=12304) 2024-11-23T15:42:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-23T15:42:49,768 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/1453e986da1b4f99bc5ee88d6cfbbdad as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/1453e986da1b4f99bc5ee88d6cfbbdad 2024-11-23T15:42:49,771 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f77eb2e3d7d1fc2f4113a5c7f8d27655/A of f77eb2e3d7d1fc2f4113a5c7f8d27655 into 1453e986da1b4f99bc5ee88d6cfbbdad(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:49,771 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:49,771 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., storeName=f77eb2e3d7d1fc2f4113a5c7f8d27655/A, priority=11, startTime=1732376569334; duration=0sec 2024-11-23T15:42:49,771 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:49,771 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f77eb2e3d7d1fc2f4113a5c7f8d27655:A 2024-11-23T15:42:49,778 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/65f8d0d38b464c5f8464edd7c4d8c8c8 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/65f8d0d38b464c5f8464edd7c4d8c8c8 2024-11-23T15:42:49,781 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f77eb2e3d7d1fc2f4113a5c7f8d27655/C of f77eb2e3d7d1fc2f4113a5c7f8d27655 into 65f8d0d38b464c5f8464edd7c4d8c8c8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:49,782 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:49,782 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., storeName=f77eb2e3d7d1fc2f4113a5c7f8d27655/C, priority=11, startTime=1732376569334; duration=0sec 2024-11-23T15:42:49,782 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:49,782 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f77eb2e3d7d1fc2f4113a5c7f8d27655:C 2024-11-23T15:42:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:49,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:49,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:49,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376629858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:49,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:49,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:49,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:49,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376629860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:49,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:49,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:49,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:49,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376629860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:49,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376629861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:49,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376629864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:49,866 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_2' in f77eb2e3d7d1fc2f4113a5c7f8d27655 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:42:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:49,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:49,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:49,893 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112307403055b35847ad86c9c0a99610f488_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112307403055b35847ad86c9c0a99610f488_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:49,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/57949d88ab9b46579286a274916ef3f0, store: [table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:49,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/57949d88ab9b46579286a274916ef3f0 is 175, key is test_row_0/A:col10/1732376567694/Put/seqid=0 2024-11-23T15:42:49,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742371_1547 (size=31105) 2024-11-23T15:42:49,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376629964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:49,966 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:49,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:49,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376629974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:49,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376629974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:49,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376629974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:49,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:50,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:50,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376630174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:50,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:50,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:50,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376630184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:50,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:50,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376630184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:50,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:50,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376630184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:50,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:50,298 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/57949d88ab9b46579286a274916ef3f0 2024-11-23T15:42:50,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/3fb5e308c2e646d989c251c46d1b9876 is 50, key is test_row_0/B:col10/1732376567694/Put/seqid=0 2024-11-23T15:42:50,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742372_1548 (size=12151) 2024-11-23T15:42:50,307 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/3fb5e308c2e646d989c251c46d1b9876 2024-11-23T15:42:50,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/f364ca57f8934091a37121233ce5e76f is 50, key is test_row_0/C:col10/1732376567694/Put/seqid=0 2024-11-23T15:42:50,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742373_1549 (size=12151) 2024-11-23T15:42:50,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:50,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376630484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:50,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376630494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:50,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376630494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:50,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:50,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:50,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:50,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:50,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376630494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:50,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:50,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:50,497 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:50,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-23T15:42:50,717 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/f364ca57f8934091a37121233ce5e76f 2024-11-23T15:42:50,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/57949d88ab9b46579286a274916ef3f0 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/57949d88ab9b46579286a274916ef3f0 2024-11-23T15:42:50,723 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/57949d88ab9b46579286a274916ef3f0, entries=150, sequenceid=153, filesize=30.4 K 2024-11-23T15:42:50,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/3fb5e308c2e646d989c251c46d1b9876 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/3fb5e308c2e646d989c251c46d1b9876 2024-11-23T15:42:50,726 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/3fb5e308c2e646d989c251c46d1b9876, entries=150, sequenceid=153, filesize=11.9 K 2024-11-23T15:42:50,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/f364ca57f8934091a37121233ce5e76f as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f364ca57f8934091a37121233ce5e76f 2024-11-23T15:42:50,729 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f364ca57f8934091a37121233ce5e76f, entries=150, sequenceid=153, filesize=11.9 K 2024-11-23T15:42:50,730 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for f77eb2e3d7d1fc2f4113a5c7f8d27655 in 1250ms, sequenceid=153, compaction requested=false 2024-11-23T15:42:50,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:50,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:50,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-23T15:42:50,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-23T15:42:50,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-23T15:42:50,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3240 sec 2024-11-23T15:42:50,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.3280 sec 2024-11-23T15:42:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:50,995 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:50,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-23T15:42:51,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112355a49ca3fc7a40bbba37d69a0d2ea2ca_f77eb2e3d7d1fc2f4113a5c7f8d27655 is 50, key is test_row_0/A:col10/1732376569859/Put/seqid=0 2024-11-23T15:42:51,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742374_1550 (size=14794) 2024-11-23T15:42:51,004 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:51,007 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112355a49ca3fc7a40bbba37d69a0d2ea2ca_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112355a49ca3fc7a40bbba37d69a0d2ea2ca_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:51,008 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/8bae3fbee2e84ef5b836f84bc0dbf9c8, store: [table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:51,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/8bae3fbee2e84ef5b836f84bc0dbf9c8 is 175, key is test_row_0/A:col10/1732376569859/Put/seqid=0 2024-11-23T15:42:51,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742375_1551 (size=39749) 2024-11-23T15:42:51,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376631017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:51,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376631020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:51,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:51,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376631020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:51,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376631021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:51,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:51,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:51,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376631124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:51,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:51,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:51,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376631134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:51,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:51,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376631134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:51,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:51,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:51,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376631134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:51,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:51,137 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,137 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,137 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:51,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376631334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:51,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:51,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376631344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:51,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376631344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376631344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:51,414 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/8bae3fbee2e84ef5b836f84bc0dbf9c8 2024-11-23T15:42:51,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/101a4b0a9dfe451b94cf3c720ada4b92 is 50, key is test_row_0/B:col10/1732376569859/Put/seqid=0 2024-11-23T15:42:51,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742376_1552 (size=12151) 2024-11-23T15:42:51,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376631644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:51,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:51,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376631654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:51,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:51,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376631654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:51,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:51,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376631654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:51,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,658 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:51,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/101a4b0a9dfe451b94cf3c720ada4b92 2024-11-23T15:42:51,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/922cb22a07434c91b02209335350a94c is 50, key is test_row_0/C:col10/1732376569859/Put/seqid=0 2024-11-23T15:42:51,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742377_1553 (size=12151) 2024-11-23T15:42:51,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/922cb22a07434c91b02209335350a94c 2024-11-23T15:42:51,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/8bae3fbee2e84ef5b836f84bc0dbf9c8 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/8bae3fbee2e84ef5b836f84bc0dbf9c8 2024-11-23T15:42:51,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/8bae3fbee2e84ef5b836f84bc0dbf9c8, entries=200, sequenceid=170, filesize=38.8 K 2024-11-23T15:42:51,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/101a4b0a9dfe451b94cf3c720ada4b92 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/101a4b0a9dfe451b94cf3c720ada4b92 2024-11-23T15:42:51,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/101a4b0a9dfe451b94cf3c720ada4b92, entries=150, sequenceid=170, filesize=11.9 K 2024-11-23T15:42:51,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/922cb22a07434c91b02209335350a94c as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/922cb22a07434c91b02209335350a94c 2024-11-23T15:42:51,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/922cb22a07434c91b02209335350a94c, entries=150, sequenceid=170, filesize=11.9 K 2024-11-23T15:42:51,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for f77eb2e3d7d1fc2f4113a5c7f8d27655 in 858ms, sequenceid=170, compaction requested=true 2024-11-23T15:42:51,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:51,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f77eb2e3d7d1fc2f4113a5c7f8d27655:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:42:51,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:51,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f77eb2e3d7d1fc2f4113a5c7f8d27655:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:42:51,854 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:51,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:51,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f77eb2e3d7d1fc2f4113a5c7f8d27655:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:42:51,854 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:51,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:51,855 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:51,855 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102133 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:51,855 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): f77eb2e3d7d1fc2f4113a5c7f8d27655/A is initiating minor compaction (all files) 2024-11-23T15:42:51,855 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): f77eb2e3d7d1fc2f4113a5c7f8d27655/B is initiating minor compaction (all files) 2024-11-23T15:42:51,855 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f77eb2e3d7d1fc2f4113a5c7f8d27655/A in TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:51,855 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f77eb2e3d7d1fc2f4113a5c7f8d27655/B in TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:51,855 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/b0bbdc753d154b2fbc3e45aac4575851, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/3fb5e308c2e646d989c251c46d1b9876, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/101a4b0a9dfe451b94cf3c720ada4b92] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp, totalSize=35.8 K 2024-11-23T15:42:51,855 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/1453e986da1b4f99bc5ee88d6cfbbdad, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/57949d88ab9b46579286a274916ef3f0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/8bae3fbee2e84ef5b836f84bc0dbf9c8] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp, totalSize=99.7 K 2024-11-23T15:42:51,855 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:51,855 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. files: [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/1453e986da1b4f99bc5ee88d6cfbbdad, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/57949d88ab9b46579286a274916ef3f0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/8bae3fbee2e84ef5b836f84bc0dbf9c8] 2024-11-23T15:42:51,855 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting b0bbdc753d154b2fbc3e45aac4575851, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732376566516 2024-11-23T15:42:51,855 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1453e986da1b4f99bc5ee88d6cfbbdad, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732376566516 2024-11-23T15:42:51,856 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57949d88ab9b46579286a274916ef3f0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732376567691 2024-11-23T15:42:51,856 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bae3fbee2e84ef5b836f84bc0dbf9c8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732376569855 2024-11-23T15:42:51,856 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fb5e308c2e646d989c251c46d1b9876, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732376567691 2024-11-23T15:42:51,856 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 101a4b0a9dfe451b94cf3c720ada4b92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732376569855 2024-11-23T15:42:51,860 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:51,862 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f77eb2e3d7d1fc2f4113a5c7f8d27655#B#compaction#475 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:51,862 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/6d908cd618bf4517b567a89e82f6e5e7 is 50, key is test_row_0/B:col10/1732376569859/Put/seqid=0 2024-11-23T15:42:51,862 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411236a12f9315b6c465f8c4539f662e2b462_f77eb2e3d7d1fc2f4113a5c7f8d27655 store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:51,864 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411236a12f9315b6c465f8c4539f662e2b462_f77eb2e3d7d1fc2f4113a5c7f8d27655, store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:51,864 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411236a12f9315b6c465f8c4539f662e2b462_f77eb2e3d7d1fc2f4113a5c7f8d27655 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:51,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742378_1554 (size=12527) 2024-11-23T15:42:51,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742379_1555 (size=4469) 2024-11-23T15:42:51,870 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/6d908cd618bf4517b567a89e82f6e5e7 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/6d908cd618bf4517b567a89e82f6e5e7 2024-11-23T15:42:51,873 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f77eb2e3d7d1fc2f4113a5c7f8d27655/B of f77eb2e3d7d1fc2f4113a5c7f8d27655 into 6d908cd618bf4517b567a89e82f6e5e7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:51,873 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:51,874 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., storeName=f77eb2e3d7d1fc2f4113a5c7f8d27655/B, priority=13, startTime=1732376571854; duration=0sec 2024-11-23T15:42:51,874 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:51,874 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f77eb2e3d7d1fc2f4113a5c7f8d27655:B 2024-11-23T15:42:51,874 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:51,874 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:51,874 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): f77eb2e3d7d1fc2f4113a5c7f8d27655/C is initiating minor compaction (all files) 2024-11-23T15:42:51,874 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f77eb2e3d7d1fc2f4113a5c7f8d27655/C in TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:51,874 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/65f8d0d38b464c5f8464edd7c4d8c8c8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f364ca57f8934091a37121233ce5e76f, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/922cb22a07434c91b02209335350a94c] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp, totalSize=35.8 K 2024-11-23T15:42:51,875 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 65f8d0d38b464c5f8464edd7c4d8c8c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732376566516 2024-11-23T15:42:51,875 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting f364ca57f8934091a37121233ce5e76f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732376567691 2024-11-23T15:42:51,875 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 922cb22a07434c91b02209335350a94c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732376569855 2024-11-23T15:42:51,880 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f77eb2e3d7d1fc2f4113a5c7f8d27655#C#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:51,880 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/2b4b411417b545f3b2aedef284d667d8 is 50, key is test_row_0/C:col10/1732376569859/Put/seqid=0 2024-11-23T15:42:51,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742380_1556 (size=12527) 2024-11-23T15:42:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:51,896 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:51,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-23T15:42:51,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ccd5594573fb435789a1247897a5adc2_f77eb2e3d7d1fc2f4113a5c7f8d27655 is 50, key is test_row_0/A:col10/1732376571018/Put/seqid=0 2024-11-23T15:42:51,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742381_1557 (size=14794) 2024-11-23T15:42:51,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376631952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:51,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:51,958 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:52,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:52,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376632064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:52,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:52,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376632154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:52,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:52,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:52,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:52,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376632164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:52,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376632164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:52,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376632164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:52,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:52,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:52,168 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:52,269 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f77eb2e3d7d1fc2f4113a5c7f8d27655#A#compaction#474 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:52,269 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/72fa4d54e6b041999fed7108b73ab8e2 is 175, key is test_row_0/A:col10/1732376569859/Put/seqid=0 2024-11-23T15:42:52,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742382_1558 (size=31481) 2024-11-23T15:42:52,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376632274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:52,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:52,287 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/2b4b411417b545f3b2aedef284d667d8 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2b4b411417b545f3b2aedef284d667d8 2024-11-23T15:42:52,290 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f77eb2e3d7d1fc2f4113a5c7f8d27655/C of f77eb2e3d7d1fc2f4113a5c7f8d27655 into 2b4b411417b545f3b2aedef284d667d8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:52,290 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:52,290 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., storeName=f77eb2e3d7d1fc2f4113a5c7f8d27655/C, priority=13, startTime=1732376571854; duration=0sec 2024-11-23T15:42:52,290 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:52,290 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f77eb2e3d7d1fc2f4113a5c7f8d27655:C 2024-11-23T15:42:52,324 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:52,327 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ccd5594573fb435789a1247897a5adc2_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ccd5594573fb435789a1247897a5adc2_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:52,327 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/792d122d7caa465a8a3b3e3ff166e795, store: [table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:52,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/792d122d7caa465a8a3b3e3ff166e795 is 175, key is test_row_0/A:col10/1732376571018/Put/seqid=0 2024-11-23T15:42:52,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742383_1559 (size=39749) 2024-11-23T15:42:52,334 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/792d122d7caa465a8a3b3e3ff166e795 2024-11-23T15:42:52,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/a65bc193cf434df09d4f202bdc65c0a9 is 50, key is test_row_0/B:col10/1732376571018/Put/seqid=0 2024-11-23T15:42:52,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742384_1560 (size=12151) 2024-11-23T15:42:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-23T15:42:52,575 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:52,575 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:42:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-23T15:42:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-23T15:42:52,577 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:42:52,578 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:42:52,578 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:42:52,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376632584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:52,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:52,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:52,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:52,676 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/72fa4d54e6b041999fed7108b73ab8e2 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/72fa4d54e6b041999fed7108b73ab8e2 2024-11-23T15:42:52,680 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f77eb2e3d7d1fc2f4113a5c7f8d27655/A of f77eb2e3d7d1fc2f4113a5c7f8d27655 into 72fa4d54e6b041999fed7108b73ab8e2(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:52,680 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:52,680 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., storeName=f77eb2e3d7d1fc2f4113a5c7f8d27655/A, priority=13, startTime=1732376571854; duration=0sec 2024-11-23T15:42:52,680 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:52,680 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f77eb2e3d7d1fc2f4113a5c7f8d27655:A 2024-11-23T15:42:52,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-23T15:42:52,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-23T15:42:52,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:52,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:52,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:52,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:52,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:52,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:52,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/a65bc193cf434df09d4f202bdc65c0a9 2024-11-23T15:42:52,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/8eef402c7ea74980ae6c4f49c615c2f5 is 50, key is test_row_0/C:col10/1732376571018/Put/seqid=0 2024-11-23T15:42:52,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742385_1561 (size=12151) 2024-11-23T15:42:52,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-23T15:42:52,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:52,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:52,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:52,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:52,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:52,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-23T15:42:53,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-23T15:42:53,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:53,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:53,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:53,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:53,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:53,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:53,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:53,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376633094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:53,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,096 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:53,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/8eef402c7ea74980ae6c4f49c615c2f5 2024-11-23T15:42:53,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/792d122d7caa465a8a3b3e3ff166e795 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/792d122d7caa465a8a3b3e3ff166e795 2024-11-23T15:42:53,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/792d122d7caa465a8a3b3e3ff166e795, entries=200, sequenceid=193, filesize=38.8 K 2024-11-23T15:42:53,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/a65bc193cf434df09d4f202bdc65c0a9 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/a65bc193cf434df09d4f202bdc65c0a9 2024-11-23T15:42:53,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/a65bc193cf434df09d4f202bdc65c0a9, entries=150, sequenceid=193, filesize=11.9 K 2024-11-23T15:42:53,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/8eef402c7ea74980ae6c4f49c615c2f5 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/8eef402c7ea74980ae6c4f49c615c2f5 2024-11-23T15:42:53,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/8eef402c7ea74980ae6c4f49c615c2f5, entries=150, sequenceid=193, filesize=11.9 K 2024-11-23T15:42:53,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for f77eb2e3d7d1fc2f4113a5c7f8d27655 in 1274ms, sequenceid=193, compaction requested=false 2024-11-23T15:42:53,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:53,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:53,177 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:53,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-23T15:42:53,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ea35209dacc741fe84a1911726c97542_f77eb2e3d7d1fc2f4113a5c7f8d27655 is 50, key is test_row_0/A:col10/1732376573176/Put/seqid=0 2024-11-23T15:42:53,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-23T15:42:53,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:53,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:53,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:53,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742386_1562 (size=14794) 2024-11-23T15:42:53,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:53,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:53,188 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:53,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:53,191 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ea35209dacc741fe84a1911726c97542_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ea35209dacc741fe84a1911726c97542_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:53,192 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/413eff813c8448f3b171a92f9205f317, store: [table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:53,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/413eff813c8448f3b171a92f9205f317 is 175, key is test_row_0/A:col10/1732376573176/Put/seqid=0 2024-11-23T15:42:53,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-23T15:42:53,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:53,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376633202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:53,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:53,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:53,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376633202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:53,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376633202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:53,207 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:53,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,207 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:53,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:53,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742387_1563 (size=39749) 2024-11-23T15:42:53,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:53,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376633203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:53,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376633314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:53,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376633314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376633314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:53,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376633314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,319 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:53,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-23T15:42:53,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:53,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:53,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:53,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:53,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:53,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:53,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-23T15:42:53,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:53,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:53,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:53,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:53,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:53,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:53,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:53,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376633524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376633524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:53,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376633524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376633525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:53,609 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/413eff813c8448f3b171a92f9205f317 2024-11-23T15:42:53,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/9d1575edbbc146d1a51554077b952666 is 50, key is test_row_0/B:col10/1732376573176/Put/seqid=0 2024-11-23T15:42:53,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742388_1564 (size=12151) 2024-11-23T15:42:53,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-23T15:42:53,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:53,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:53,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:53,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:53,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:53,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-23T15:42:53,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-23T15:42:53,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:53,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:53,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:53,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:53,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:53,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376633834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:53,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376633834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376633835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:53,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376633835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:53,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-23T15:42:53,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:53,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:53,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:53,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:53,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:54,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/9d1575edbbc146d1a51554077b952666 2024-11-23T15:42:54,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/d5b99dfc4107427a84dd2ec5bcc6afe9 is 50, key is test_row_0/C:col10/1732376573176/Put/seqid=0 2024-11-23T15:42:54,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742389_1565 (size=12151) 2024-11-23T15:42:54,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/d5b99dfc4107427a84dd2ec5bcc6afe9 2024-11-23T15:42:54,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/413eff813c8448f3b171a92f9205f317 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/413eff813c8448f3b171a92f9205f317 2024-11-23T15:42:54,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/413eff813c8448f3b171a92f9205f317, entries=200, sequenceid=211, filesize=38.8 K 2024-11-23T15:42:54,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/9d1575edbbc146d1a51554077b952666 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/9d1575edbbc146d1a51554077b952666 2024-11-23T15:42:54,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/9d1575edbbc146d1a51554077b952666, entries=150, sequenceid=211, filesize=11.9 K 2024-11-23T15:42:54,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/d5b99dfc4107427a84dd2ec5bcc6afe9 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/d5b99dfc4107427a84dd2ec5bcc6afe9 2024-11-23T15:42:54,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/d5b99dfc4107427a84dd2ec5bcc6afe9, entries=150, sequenceid=211, filesize=11.9 K 2024-11-23T15:42:54,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for f77eb2e3d7d1fc2f4113a5c7f8d27655 in 904ms, sequenceid=211, compaction requested=true 2024-11-23T15:42:54,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:54,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f77eb2e3d7d1fc2f4113a5c7f8d27655:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:42:54,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:54,082 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:54,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f77eb2e3d7d1fc2f4113a5c7f8d27655:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:42:54,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:54,082 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:54,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f77eb2e3d7d1fc2f4113a5c7f8d27655:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:42:54,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:54,083 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110979 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:54,083 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): f77eb2e3d7d1fc2f4113a5c7f8d27655/A is initiating minor compaction (all files) 2024-11-23T15:42:54,083 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:54,083 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): f77eb2e3d7d1fc2f4113a5c7f8d27655/B is initiating minor compaction (all files) 2024-11-23T15:42:54,083 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f77eb2e3d7d1fc2f4113a5c7f8d27655/A in TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:54,083 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f77eb2e3d7d1fc2f4113a5c7f8d27655/B in TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:54,083 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/72fa4d54e6b041999fed7108b73ab8e2, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/792d122d7caa465a8a3b3e3ff166e795, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/413eff813c8448f3b171a92f9205f317] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp, totalSize=108.4 K 2024-11-23T15:42:54,083 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/6d908cd618bf4517b567a89e82f6e5e7, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/a65bc193cf434df09d4f202bdc65c0a9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/9d1575edbbc146d1a51554077b952666] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp, totalSize=36.0 K 2024-11-23T15:42:54,083 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:54,083 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. files: [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/72fa4d54e6b041999fed7108b73ab8e2, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/792d122d7caa465a8a3b3e3ff166e795, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/413eff813c8448f3b171a92f9205f317] 2024-11-23T15:42:54,083 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d908cd618bf4517b567a89e82f6e5e7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732376569855 2024-11-23T15:42:54,084 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72fa4d54e6b041999fed7108b73ab8e2, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732376569855 2024-11-23T15:42:54,084 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting a65bc193cf434df09d4f202bdc65c0a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732376571014 2024-11-23T15:42:54,085 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d1575edbbc146d1a51554077b952666, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732376571924 2024-11-23T15:42:54,085 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 792d122d7caa465a8a3b3e3ff166e795, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732376571014 2024-11-23T15:42:54,085 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 413eff813c8448f3b171a92f9205f317, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732376571924 2024-11-23T15:42:54,095 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:54,104 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f77eb2e3d7d1fc2f4113a5c7f8d27655#B#compaction#484 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:54,104 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/a67b8c040a454dcc9173bc6790486d0e is 50, key is test_row_0/B:col10/1732376573176/Put/seqid=0 2024-11-23T15:42:54,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-23T15:42:54,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:54,106 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-23T15:42:54,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:54,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:54,119 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123d0e6d1e6678645f786bdecedc3add126_f77eb2e3d7d1fc2f4113a5c7f8d27655 store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:54,121 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123d0e6d1e6678645f786bdecedc3add126_f77eb2e3d7d1fc2f4113a5c7f8d27655, store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:54,121 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123d0e6d1e6678645f786bdecedc3add126_f77eb2e3d7d1fc2f4113a5c7f8d27655 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:54,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742390_1566 (size=12629) 2024-11-23T15:42:54,144 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/a67b8c040a454dcc9173bc6790486d0e as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/a67b8c040a454dcc9173bc6790486d0e 2024-11-23T15:42:54,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123575a1c7c6d1f4da7968622b53b43a07c_f77eb2e3d7d1fc2f4113a5c7f8d27655 is 50, key is test_row_0/A:col10/1732376573202/Put/seqid=0 2024-11-23T15:42:54,151 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f77eb2e3d7d1fc2f4113a5c7f8d27655/B of f77eb2e3d7d1fc2f4113a5c7f8d27655 into a67b8c040a454dcc9173bc6790486d0e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:54,151 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:54,151 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., storeName=f77eb2e3d7d1fc2f4113a5c7f8d27655/B, priority=13, startTime=1732376574082; duration=0sec 2024-11-23T15:42:54,152 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:54,152 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f77eb2e3d7d1fc2f4113a5c7f8d27655:B 2024-11-23T15:42:54,153 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:54,154 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:54,154 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): f77eb2e3d7d1fc2f4113a5c7f8d27655/C is initiating minor compaction (all files) 2024-11-23T15:42:54,154 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f77eb2e3d7d1fc2f4113a5c7f8d27655/C in TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:54,154 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2b4b411417b545f3b2aedef284d667d8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/8eef402c7ea74980ae6c4f49c615c2f5, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/d5b99dfc4107427a84dd2ec5bcc6afe9] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp, totalSize=36.0 K 2024-11-23T15:42:54,154 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b4b411417b545f3b2aedef284d667d8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1732376569855 2024-11-23T15:42:54,155 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8eef402c7ea74980ae6c4f49c615c2f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732376571014 2024-11-23T15:42:54,155 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting d5b99dfc4107427a84dd2ec5bcc6afe9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732376571924 2024-11-23T15:42:54,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742391_1567 (size=4469) 2024-11-23T15:42:54,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742392_1568 (size=12304) 2024-11-23T15:42:54,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:54,178 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123575a1c7c6d1f4da7968622b53b43a07c_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123575a1c7c6d1f4da7968622b53b43a07c_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:54,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/529cbd6eb46c4f4b9c0ee054813ead1e, store: [table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:54,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/529cbd6eb46c4f4b9c0ee054813ead1e is 175, key is test_row_0/A:col10/1732376573202/Put/seqid=0 2024-11-23T15:42:54,184 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f77eb2e3d7d1fc2f4113a5c7f8d27655#C#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:54,185 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/8b0bf8092c31463e9ebf5cd54aac933d is 50, key is test_row_0/C:col10/1732376573176/Put/seqid=0 2024-11-23T15:42:54,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742393_1569 (size=31105) 2024-11-23T15:42:54,187 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/529cbd6eb46c4f4b9c0ee054813ead1e 2024-11-23T15:42:54,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742394_1570 (size=12629) 2024-11-23T15:42:54,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:54,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376634188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:54,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,193 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:54,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/a69f35ea3ae44389af13a96a0b27f86d is 50, key is test_row_0/B:col10/1732376573202/Put/seqid=0 2024-11-23T15:42:54,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742395_1571 (size=12151) 2024-11-23T15:42:54,201 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/a69f35ea3ae44389af13a96a0b27f86d 2024-11-23T15:42:54,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/2bd0352a88cc4391826f044f1694806d is 50, key is test_row_0/C:col10/1732376573202/Put/seqid=0 2024-11-23T15:42:54,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742396_1572 (size=12151) 2024-11-23T15:42:54,217 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/2bd0352a88cc4391826f044f1694806d 2024-11-23T15:42:54,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/529cbd6eb46c4f4b9c0ee054813ead1e as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/529cbd6eb46c4f4b9c0ee054813ead1e 2024-11-23T15:42:54,227 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/529cbd6eb46c4f4b9c0ee054813ead1e, entries=150, sequenceid=231, filesize=30.4 K 2024-11-23T15:42:54,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/a69f35ea3ae44389af13a96a0b27f86d as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/a69f35ea3ae44389af13a96a0b27f86d 2024-11-23T15:42:54,231 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/a69f35ea3ae44389af13a96a0b27f86d, entries=150, sequenceid=231, filesize=11.9 K 2024-11-23T15:42:54,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/2bd0352a88cc4391826f044f1694806d as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2bd0352a88cc4391826f044f1694806d 2024-11-23T15:42:54,235 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2bd0352a88cc4391826f044f1694806d, entries=150, sequenceid=231, filesize=11.9 K 2024-11-23T15:42:54,235 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for f77eb2e3d7d1fc2f4113a5c7f8d27655 in 129ms, sequenceid=231, compaction requested=false 2024-11-23T15:42:54,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:54,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:54,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-23T15:42:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-23T15:42:54,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-23T15:42:54,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6590 sec 2024-11-23T15:42:54,241 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.6640 sec 2024-11-23T15:42:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:54,306 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:54,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-23T15:42:54,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112326fdbe2675de4817bfc1558b9c67260e_f77eb2e3d7d1fc2f4113a5c7f8d27655 is 50, key is test_row_0/A:col10/1732376574179/Put/seqid=0 2024-11-23T15:42:54,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742397_1573 (size=14794) 2024-11-23T15:42:54,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376634352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:54,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:54,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376634352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376634353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:54,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:54,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376634356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:54,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:54,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376634357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:54,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:54,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:54,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376634464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376634464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:54,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376634464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:54,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376634464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376634464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:54,468 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,468 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,468 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:54,559 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f77eb2e3d7d1fc2f4113a5c7f8d27655#A#compaction#483 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:54,559 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/36986cfb216b41cf83f76e3b157d6c88 is 175, key is test_row_0/A:col10/1732376573176/Put/seqid=0 2024-11-23T15:42:54,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742398_1574 (size=31583) 2024-11-23T15:42:54,604 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/8b0bf8092c31463e9ebf5cd54aac933d as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/8b0bf8092c31463e9ebf5cd54aac933d 2024-11-23T15:42:54,608 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f77eb2e3d7d1fc2f4113a5c7f8d27655/C of f77eb2e3d7d1fc2f4113a5c7f8d27655 into 8b0bf8092c31463e9ebf5cd54aac933d(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:54,608 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:54,608 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., storeName=f77eb2e3d7d1fc2f4113a5c7f8d27655/C, priority=13, startTime=1732376574082; duration=0sec 2024-11-23T15:42:54,608 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:54,609 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f77eb2e3d7d1fc2f4113a5c7f8d27655:C 2024-11-23T15:42:54,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376634674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:54,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376634674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:54,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376634674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376634674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:54,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376634674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-23T15:42:54,724 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:54,724 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:42:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-23T15:42:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-23T15:42:54,726 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:42:54,726 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:42:54,726 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:42:54,740 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:54,743 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112326fdbe2675de4817bfc1558b9c67260e_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112326fdbe2675de4817bfc1558b9c67260e_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:54,744 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/27f0af9ded4e4972b84de9186943e773, store: [table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:54,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/27f0af9ded4e4972b84de9186943e773 is 175, key is test_row_0/A:col10/1732376574179/Put/seqid=0 2024-11-23T15:42:54,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742399_1575 (size=39749) 2024-11-23T15:42:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-23T15:42:54,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-23T15:42:54,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:54,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:54,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:54,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:54,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:54,966 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/36986cfb216b41cf83f76e3b157d6c88 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/36986cfb216b41cf83f76e3b157d6c88 2024-11-23T15:42:54,969 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f77eb2e3d7d1fc2f4113a5c7f8d27655/A of f77eb2e3d7d1fc2f4113a5c7f8d27655 into 36986cfb216b41cf83f76e3b157d6c88(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:54,969 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:54,969 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., storeName=f77eb2e3d7d1fc2f4113a5c7f8d27655/A, priority=13, startTime=1732376574082; duration=0sec 2024-11-23T15:42:54,970 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:54,970 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f77eb2e3d7d1fc2f4113a5c7f8d27655:A 2024-11-23T15:42:54,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:54,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376634984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:54,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376634984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:54,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:54,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:54,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376634984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:54,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:54,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,986 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:54,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376634984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:54,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:54,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376634984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:54,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:54,987 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,987 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:54,987 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:55,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-23T15:42:55,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:55,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:55,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:55,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:55,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:55,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-23T15:42:55,147 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/27f0af9ded4e4972b84de9186943e773 2024-11-23T15:42:55,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/06a38dedf25e4f4e99c647a6e2948186 is 50, key is test_row_0/B:col10/1732376574179/Put/seqid=0 2024-11-23T15:42:55,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742400_1576 (size=12151) 2024-11-23T15:42:55,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/06a38dedf25e4f4e99c647a6e2948186 2024-11-23T15:42:55,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/f5d5ca7800c74816a46f99d70702afac is 50, key is test_row_0/C:col10/1732376574179/Put/seqid=0 2024-11-23T15:42:55,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742401_1577 (size=12151) 2024-11-23T15:42:55,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-23T15:42:55,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:55,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:55,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:55,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:55,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:55,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-23T15:42:55,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:55,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:55,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:55,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:55,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:55,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-23T15:42:55,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-23T15:42:55,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:55,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:55,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:55,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:55,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:55,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:55,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376635494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:55,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:55,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:55,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:55,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376635494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:55,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:55,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:55,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:55,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:55,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:55,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376635495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:55,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:55,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:55,497 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376635495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:55,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:55,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376635495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:55,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:55,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:55,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:55,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:55,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:55,498 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:55,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/f5d5ca7800c74816a46f99d70702afac 2024-11-23T15:42:55,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/27f0af9ded4e4972b84de9186943e773 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/27f0af9ded4e4972b84de9186943e773 2024-11-23T15:42:55,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/27f0af9ded4e4972b84de9186943e773, entries=200, sequenceid=249, filesize=38.8 K 2024-11-23T15:42:55,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/06a38dedf25e4f4e99c647a6e2948186 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/06a38dedf25e4f4e99c647a6e2948186 2024-11-23T15:42:55,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/06a38dedf25e4f4e99c647a6e2948186, entries=150, sequenceid=249, filesize=11.9 K 2024-11-23T15:42:55,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/f5d5ca7800c74816a46f99d70702afac as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f5d5ca7800c74816a46f99d70702afac 2024-11-23T15:42:55,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f5d5ca7800c74816a46f99d70702afac, entries=150, sequenceid=249, filesize=11.9 K 2024-11-23T15:42:55,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for f77eb2e3d7d1fc2f4113a5c7f8d27655 in 1288ms, sequenceid=249, compaction requested=true 2024-11-23T15:42:55,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:55,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f77eb2e3d7d1fc2f4113a5c7f8d27655:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:42:55,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:55,594 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:55,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f77eb2e3d7d1fc2f4113a5c7f8d27655:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:42:55,594 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:55,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:55,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f77eb2e3d7d1fc2f4113a5c7f8d27655:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:42:55,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:55,595 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:55,595 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:55,595 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): f77eb2e3d7d1fc2f4113a5c7f8d27655/B is initiating minor compaction (all files) 2024-11-23T15:42:55,595 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): f77eb2e3d7d1fc2f4113a5c7f8d27655/A is initiating minor compaction (all files) 2024-11-23T15:42:55,595 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f77eb2e3d7d1fc2f4113a5c7f8d27655/B in TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:55,595 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f77eb2e3d7d1fc2f4113a5c7f8d27655/A in TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:55,595 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/a67b8c040a454dcc9173bc6790486d0e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/a69f35ea3ae44389af13a96a0b27f86d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/06a38dedf25e4f4e99c647a6e2948186] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp, totalSize=36.1 K 2024-11-23T15:42:55,595 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/36986cfb216b41cf83f76e3b157d6c88, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/529cbd6eb46c4f4b9c0ee054813ead1e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/27f0af9ded4e4972b84de9186943e773] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp, totalSize=100.0 K 2024-11-23T15:42:55,595 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:55,595 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. files: [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/36986cfb216b41cf83f76e3b157d6c88, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/529cbd6eb46c4f4b9c0ee054813ead1e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/27f0af9ded4e4972b84de9186943e773] 2024-11-23T15:42:55,595 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting a67b8c040a454dcc9173bc6790486d0e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732376571924 2024-11-23T15:42:55,595 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36986cfb216b41cf83f76e3b157d6c88, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732376571924 2024-11-23T15:42:55,595 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting a69f35ea3ae44389af13a96a0b27f86d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732376573195 2024-11-23T15:42:55,596 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 529cbd6eb46c4f4b9c0ee054813ead1e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732376573195 2024-11-23T15:42:55,596 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27f0af9ded4e4972b84de9186943e773, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732376574173 2024-11-23T15:42:55,596 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 06a38dedf25e4f4e99c647a6e2948186, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732376574173 2024-11-23T15:42:55,601 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f77eb2e3d7d1fc2f4113a5c7f8d27655#B#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:55,601 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/0d0fc429b81f40b3baf293aeff678134 is 50, key is test_row_0/B:col10/1732376574179/Put/seqid=0 2024-11-23T15:42:55,602 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:55,612 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112332e47b0f773f4380858a6bc4e2360bce_f77eb2e3d7d1fc2f4113a5c7f8d27655 store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:55,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742402_1578 (size=12731) 2024-11-23T15:42:55,614 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112332e47b0f773f4380858a6bc4e2360bce_f77eb2e3d7d1fc2f4113a5c7f8d27655, store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:55,614 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112332e47b0f773f4380858a6bc4e2360bce_f77eb2e3d7d1fc2f4113a5c7f8d27655 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:55,619 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/0d0fc429b81f40b3baf293aeff678134 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/0d0fc429b81f40b3baf293aeff678134 2024-11-23T15:42:55,623 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f77eb2e3d7d1fc2f4113a5c7f8d27655/B of f77eb2e3d7d1fc2f4113a5c7f8d27655 into 0d0fc429b81f40b3baf293aeff678134(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:55,623 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:55,623 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., storeName=f77eb2e3d7d1fc2f4113a5c7f8d27655/B, priority=13, startTime=1732376575594; duration=0sec 2024-11-23T15:42:55,623 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:55,623 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f77eb2e3d7d1fc2f4113a5c7f8d27655:B 2024-11-23T15:42:55,623 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:55,624 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:55,624 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): f77eb2e3d7d1fc2f4113a5c7f8d27655/C is initiating minor compaction (all files) 2024-11-23T15:42:55,624 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f77eb2e3d7d1fc2f4113a5c7f8d27655/C in TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:55,624 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/8b0bf8092c31463e9ebf5cd54aac933d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2bd0352a88cc4391826f044f1694806d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f5d5ca7800c74816a46f99d70702afac] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp, totalSize=36.1 K 2024-11-23T15:42:55,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742403_1579 (size=4469) 2024-11-23T15:42:55,625 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b0bf8092c31463e9ebf5cd54aac933d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732376571924 2024-11-23T15:42:55,625 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bd0352a88cc4391826f044f1694806d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732376573195 2024-11-23T15:42:55,625 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting f5d5ca7800c74816a46f99d70702afac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732376574173 2024-11-23T15:42:55,625 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f77eb2e3d7d1fc2f4113a5c7f8d27655#A#compaction#493 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:55,626 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/6951b51490fd465f97b1c7be6782c809 is 175, key is test_row_0/A:col10/1732376574179/Put/seqid=0 2024-11-23T15:42:55,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742404_1580 (size=31685) 2024-11-23T15:42:55,631 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f77eb2e3d7d1fc2f4113a5c7f8d27655#C#compaction#494 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:55,631 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/a6ecfbe8f8f5401a9d312618835117df is 50, key is test_row_0/C:col10/1732376574179/Put/seqid=0 2024-11-23T15:42:55,634 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/6951b51490fd465f97b1c7be6782c809 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/6951b51490fd465f97b1c7be6782c809 2024-11-23T15:42:55,638 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f77eb2e3d7d1fc2f4113a5c7f8d27655/A of f77eb2e3d7d1fc2f4113a5c7f8d27655 into 6951b51490fd465f97b1c7be6782c809(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:55,638 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:55,638 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., storeName=f77eb2e3d7d1fc2f4113a5c7f8d27655/A, priority=13, startTime=1732376575594; duration=0sec 2024-11-23T15:42:55,638 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:55,638 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f77eb2e3d7d1fc2f4113a5c7f8d27655:A 2024-11-23T15:42:55,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742405_1581 (size=12731) 2024-11-23T15:42:55,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-23T15:42:55,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:55,642 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-23T15:42:55,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123dc8aa8f79078452d8565e428233e036d_f77eb2e3d7d1fc2f4113a5c7f8d27655 is 50, key is test_row_0/A:col10/1732376574344/Put/seqid=0 2024-11-23T15:42:55,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742406_1582 (size=12454) 2024-11-23T15:42:55,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:55,654 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123dc8aa8f79078452d8565e428233e036d_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123dc8aa8f79078452d8565e428233e036d_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:55,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/0c80e750f24d433cb0151d75f410b9a8, store: [table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:55,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/0c80e750f24d433cb0151d75f410b9a8 is 175, key is test_row_0/A:col10/1732376574344/Put/seqid=0 2024-11-23T15:42:55,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742407_1583 (size=31255) 2024-11-23T15:42:55,659 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/0c80e750f24d433cb0151d75f410b9a8 2024-11-23T15:42:55,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/e55882e55e7d4fc688f82a492da92389 is 50, key is test_row_0/B:col10/1732376574344/Put/seqid=0 2024-11-23T15:42:55,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742408_1584 (size=12301) 2024-11-23T15:42:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-23T15:42:56,043 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/a6ecfbe8f8f5401a9d312618835117df as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/a6ecfbe8f8f5401a9d312618835117df 2024-11-23T15:42:56,047 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f77eb2e3d7d1fc2f4113a5c7f8d27655/C of f77eb2e3d7d1fc2f4113a5c7f8d27655 into a6ecfbe8f8f5401a9d312618835117df(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:56,047 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:56,047 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., storeName=f77eb2e3d7d1fc2f4113a5c7f8d27655/C, priority=13, startTime=1732376575594; duration=0sec 2024-11-23T15:42:56,047 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:56,047 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f77eb2e3d7d1fc2f4113a5c7f8d27655:C 2024-11-23T15:42:56,076 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/e55882e55e7d4fc688f82a492da92389 2024-11-23T15:42:56,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/245b13c41e204575aeb48a01de90328d is 50, key is test_row_0/C:col10/1732376574344/Put/seqid=0 2024-11-23T15:42:56,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742409_1585 (size=12301) 2024-11-23T15:42:56,485 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/245b13c41e204575aeb48a01de90328d 2024-11-23T15:42:56,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/0c80e750f24d433cb0151d75f410b9a8 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/0c80e750f24d433cb0151d75f410b9a8 2024-11-23T15:42:56,492 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/0c80e750f24d433cb0151d75f410b9a8, entries=150, sequenceid=272, filesize=30.5 K 2024-11-23T15:42:56,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/e55882e55e7d4fc688f82a492da92389 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/e55882e55e7d4fc688f82a492da92389 2024-11-23T15:42:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,495 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/e55882e55e7d4fc688f82a492da92389, entries=150, sequenceid=272, filesize=12.0 K 2024-11-23T15:42:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/245b13c41e204575aeb48a01de90328d as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/245b13c41e204575aeb48a01de90328d 2024-11-23T15:42:56,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,498 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/245b13c41e204575aeb48a01de90328d, entries=150, sequenceid=272, filesize=12.0 K 2024-11-23T15:42:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,499 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for f77eb2e3d7d1fc2f4113a5c7f8d27655 in 857ms, sequenceid=272, compaction requested=false 2024-11-23T15:42:56,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-23T15:42:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-23T15:42:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-23T15:42:56,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7740 sec 2024-11-23T15:42:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.7780 sec 2024-11-23T15:42:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:56,542 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:56,542 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T15:42:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411235dc829bd57d94b05b3f942803149a63a_f77eb2e3d7d1fc2f4113a5c7f8d27655 is 50, key is test_row_1/A:col10/1732376576540/Put/seqid=0 2024-11-23T15:42:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742410_1586 (size=17534) 2024-11-23T15:42:56,568 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:56,572 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411235dc829bd57d94b05b3f942803149a63a_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235dc829bd57d94b05b3f942803149a63a_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:56,573 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/d2e76ee658da4472898c22d31645fc52, store: [table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:56,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/d2e76ee658da4472898c22d31645fc52 is 175, key is test_row_1/A:col10/1732376576540/Put/seqid=0 2024-11-23T15:42:56,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742411_1587 (size=48635) 2024-11-23T15:42:56,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:56,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376636576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:56,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:56,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:56,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376636577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:56,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:56,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:56,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376636577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:56,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:56,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376636578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:56,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:56,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:56,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376636578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:56,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:56,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:56,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376636684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:56,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:56,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376636684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:56,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:56,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376636694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376636694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:56,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376636694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:56,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-23T15:42:56,874 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:56,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:42:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-23T15:42:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-23T15:42:56,876 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:42:56,877 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:42:56,877 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:42:56,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:56,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376636894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:56,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:56,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376636895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:56,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:56,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376636904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:56,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376636904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:56,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376636904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:56,906 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:56,979 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/d2e76ee658da4472898c22d31645fc52 2024-11-23T15:42:56,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-23T15:42:56,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/d0944a7b951742c687c34ce22e041595 is 50, key is test_row_1/B:col10/1732376576540/Put/seqid=0 2024-11-23T15:42:56,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742412_1588 (size=9857) 2024-11-23T15:42:56,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/d0944a7b951742c687c34ce22e041595 2024-11-23T15:42:56,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/07f28809e3b84e8d84921ad3d831c20d is 50, key is test_row_1/C:col10/1732376576540/Put/seqid=0 2024-11-23T15:42:56,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742413_1589 (size=9857) 2024-11-23T15:42:57,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-23T15:42:57,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:57,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:57,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:57,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:57,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:57,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:57,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-23T15:42:57,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:57,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:57,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:57,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:57,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-23T15:42:57,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376637203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:57,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:57,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376637204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:57,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:57,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:57,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:57,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376637214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376637214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:57,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:57,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376637214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:57,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:57,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:57,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,217 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:57,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-23T15:42:57,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:57,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:57,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:57,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:57,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:57,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/07f28809e3b84e8d84921ad3d831c20d 2024-11-23T15:42:57,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/d2e76ee658da4472898c22d31645fc52 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/d2e76ee658da4472898c22d31645fc52 2024-11-23T15:42:57,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/d2e76ee658da4472898c22d31645fc52, entries=250, sequenceid=284, filesize=47.5 K 2024-11-23T15:42:57,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/d0944a7b951742c687c34ce22e041595 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/d0944a7b951742c687c34ce22e041595 2024-11-23T15:42:57,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/d0944a7b951742c687c34ce22e041595, entries=100, sequenceid=284, filesize=9.6 K 2024-11-23T15:42:57,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/07f28809e3b84e8d84921ad3d831c20d as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/07f28809e3b84e8d84921ad3d831c20d 2024-11-23T15:42:57,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/07f28809e3b84e8d84921ad3d831c20d, entries=100, sequenceid=284, filesize=9.6 K 2024-11-23T15:42:57,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f77eb2e3d7d1fc2f4113a5c7f8d27655 in 866ms, sequenceid=284, compaction requested=true 2024-11-23T15:42:57,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:57,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f77eb2e3d7d1fc2f4113a5c7f8d27655:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:42:57,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:57,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f77eb2e3d7d1fc2f4113a5c7f8d27655:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:42:57,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-23T15:42:57,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f77eb2e3d7d1fc2f4113a5c7f8d27655:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:42:57,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-23T15:42:57,410 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:57,411 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:57,411 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:57,411 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:57,411 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): f77eb2e3d7d1fc2f4113a5c7f8d27655/A is initiating minor compaction (all files) 2024-11-23T15:42:57,411 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): f77eb2e3d7d1fc2f4113a5c7f8d27655/C is initiating minor compaction (all files) 2024-11-23T15:42:57,411 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f77eb2e3d7d1fc2f4113a5c7f8d27655/A in TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:57,411 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f77eb2e3d7d1fc2f4113a5c7f8d27655/C in TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:57,412 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/6951b51490fd465f97b1c7be6782c809, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/0c80e750f24d433cb0151d75f410b9a8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/d2e76ee658da4472898c22d31645fc52] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp, totalSize=109.0 K 2024-11-23T15:42:57,412 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/a6ecfbe8f8f5401a9d312618835117df, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/245b13c41e204575aeb48a01de90328d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/07f28809e3b84e8d84921ad3d831c20d] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp, totalSize=34.1 K 2024-11-23T15:42:57,412 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:57,412 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. files: [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/6951b51490fd465f97b1c7be6782c809, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/0c80e750f24d433cb0151d75f410b9a8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/d2e76ee658da4472898c22d31645fc52] 2024-11-23T15:42:57,412 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6ecfbe8f8f5401a9d312618835117df, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732376574173 2024-11-23T15:42:57,412 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 6951b51490fd465f97b1c7be6782c809, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732376574173 2024-11-23T15:42:57,412 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 245b13c41e204575aeb48a01de90328d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732376574344 2024-11-23T15:42:57,412 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c80e750f24d433cb0151d75f410b9a8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732376574344 2024-11-23T15:42:57,412 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07f28809e3b84e8d84921ad3d831c20d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732376576540 2024-11-23T15:42:57,412 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting d2e76ee658da4472898c22d31645fc52, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732376576516 2024-11-23T15:42:57,417 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:57,418 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f77eb2e3d7d1fc2f4113a5c7f8d27655#C#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:57,418 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/b56149b55323484ba4910d8d3058ec90 is 50, key is test_row_0/C:col10/1732376574344/Put/seqid=0 2024-11-23T15:42:57,423 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411232526234e364f415bb534f9030c665147_f77eb2e3d7d1fc2f4113a5c7f8d27655 store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:57,425 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411232526234e364f415bb534f9030c665147_f77eb2e3d7d1fc2f4113a5c7f8d27655, store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:57,425 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232526234e364f415bb534f9030c665147_f77eb2e3d7d1fc2f4113a5c7f8d27655 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:57,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742414_1590 (size=12983) 2024-11-23T15:42:57,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742415_1591 (size=4469) 2024-11-23T15:42:57,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-23T15:42:57,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:57,487 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T15:42:57,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c862458a8dd74859be04e4a13e159343_f77eb2e3d7d1fc2f4113a5c7f8d27655 is 50, key is test_row_0/A:col10/1732376576576/Put/seqid=0 2024-11-23T15:42:57,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742416_1592 (size=12454) 2024-11-23T15:42:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-23T15:42:57,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:57,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:57,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:57,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376637724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:57,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:57,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376637725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:57,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:57,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376637726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:57,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376637727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:57,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376637728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:57,835 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/b56149b55323484ba4910d8d3058ec90 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/b56149b55323484ba4910d8d3058ec90 2024-11-23T15:42:57,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376637834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:57,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:57,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376637834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:57,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:57,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:57,839 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f77eb2e3d7d1fc2f4113a5c7f8d27655/C of f77eb2e3d7d1fc2f4113a5c7f8d27655 into b56149b55323484ba4910d8d3058ec90(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:57,839 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:57,839 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., storeName=f77eb2e3d7d1fc2f4113a5c7f8d27655/C, priority=13, startTime=1732376577409; duration=0sec 2024-11-23T15:42:57,839 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:57,839 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f77eb2e3d7d1fc2f4113a5c7f8d27655:C 2024-11-23T15:42:57,839 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:57,839 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:57,839 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): f77eb2e3d7d1fc2f4113a5c7f8d27655/B is initiating minor compaction (all files) 2024-11-23T15:42:57,839 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f77eb2e3d7d1fc2f4113a5c7f8d27655/B in TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:57,840 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/0d0fc429b81f40b3baf293aeff678134, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/e55882e55e7d4fc688f82a492da92389, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/d0944a7b951742c687c34ce22e041595] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp, totalSize=34.1 K 2024-11-23T15:42:57,840 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d0fc429b81f40b3baf293aeff678134, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732376574173 2024-11-23T15:42:57,840 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting e55882e55e7d4fc688f82a492da92389, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732376574344 2024-11-23T15:42:57,840 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0944a7b951742c687c34ce22e041595, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732376576540 2024-11-23T15:42:57,840 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f77eb2e3d7d1fc2f4113a5c7f8d27655#A#compaction#502 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:57,841 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/9cb054b5206b444eaab49c21d60d24d1 is 175, key is test_row_0/A:col10/1732376574344/Put/seqid=0 2024-11-23T15:42:57,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742417_1593 (size=32044) 2024-11-23T15:42:57,845 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f77eb2e3d7d1fc2f4113a5c7f8d27655#B#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:57,846 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/52999accd2814f908601f9b26f2db7dc is 50, key is test_row_0/B:col10/1732376574344/Put/seqid=0 2024-11-23T15:42:57,848 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/9cb054b5206b444eaab49c21d60d24d1 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/9cb054b5206b444eaab49c21d60d24d1 2024-11-23T15:42:57,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:57,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376637845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:57,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:57,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376637845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742418_1594 (size=12983) 2024-11-23T15:42:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:57,852 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f77eb2e3d7d1fc2f4113a5c7f8d27655/A of f77eb2e3d7d1fc2f4113a5c7f8d27655 into 9cb054b5206b444eaab49c21d60d24d1(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:57,852 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:57,852 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., storeName=f77eb2e3d7d1fc2f4113a5c7f8d27655/A, priority=13, startTime=1732376577408; duration=0sec 2024-11-23T15:42:57,852 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:57,852 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f77eb2e3d7d1fc2f4113a5c7f8d27655:A 2024-11-23T15:42:57,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:57,899 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c862458a8dd74859be04e4a13e159343_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c862458a8dd74859be04e4a13e159343_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:57,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/ce237ac8d09a4594bb5a490d16e5ff08, store: [table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:57,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/ce237ac8d09a4594bb5a490d16e5ff08 is 175, key is test_row_0/A:col10/1732376576576/Put/seqid=0 2024-11-23T15:42:57,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742419_1595 (size=31255) 2024-11-23T15:42:58,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-23T15:42:58,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:58,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376638044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:58,045 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,046 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:58,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:58,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376638044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:58,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:58,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:58,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376638054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:58,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:58,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376638055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:58,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:58,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,055 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:58,253 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/52999accd2814f908601f9b26f2db7dc as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/52999accd2814f908601f9b26f2db7dc 2024-11-23T15:42:58,256 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f77eb2e3d7d1fc2f4113a5c7f8d27655/B of f77eb2e3d7d1fc2f4113a5c7f8d27655 into 52999accd2814f908601f9b26f2db7dc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:42:58,256 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:58,256 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., storeName=f77eb2e3d7d1fc2f4113a5c7f8d27655/B, priority=13, startTime=1732376577409; duration=0sec 2024-11-23T15:42:58,256 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:58,256 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f77eb2e3d7d1fc2f4113a5c7f8d27655:B 2024-11-23T15:42:58,303 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/ce237ac8d09a4594bb5a490d16e5ff08 2024-11-23T15:42:58,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/1afb0b7577c04fdd933df1bc0b009bba is 50, key is test_row_0/B:col10/1732376576576/Put/seqid=0 2024-11-23T15:42:58,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742420_1596 (size=12301) 2024-11-23T15:42:58,312 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/1afb0b7577c04fdd933df1bc0b009bba 2024-11-23T15:42:58,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/f87f80edd94f45e1a7476fb935fdf519 is 50, key is test_row_0/C:col10/1732376576576/Put/seqid=0 2024-11-23T15:42:58,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742421_1597 (size=12301) 2024-11-23T15:42:58,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376638354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376638354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:58,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376638364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:58,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:58,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376638364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:58,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:58,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:58,723 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/f87f80edd94f45e1a7476fb935fdf519 2024-11-23T15:42:58,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/ce237ac8d09a4594bb5a490d16e5ff08 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/ce237ac8d09a4594bb5a490d16e5ff08 2024-11-23T15:42:58,729 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/ce237ac8d09a4594bb5a490d16e5ff08, entries=150, sequenceid=309, filesize=30.5 K 2024-11-23T15:42:58,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/1afb0b7577c04fdd933df1bc0b009bba as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/1afb0b7577c04fdd933df1bc0b009bba 2024-11-23T15:42:58,738 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/1afb0b7577c04fdd933df1bc0b009bba, entries=150, sequenceid=309, filesize=12.0 K 2024-11-23T15:42:58,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52766 deadline: 1732376638736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:58,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/f87f80edd94f45e1a7476fb935fdf519 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f87f80edd94f45e1a7476fb935fdf519 2024-11-23T15:42:58,738 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to b712f9af2c12,46589,1732376409194 for 'test_row_1' in f77eb2e3d7d1fc2f4113a5c7f8d27655 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor45.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T15:42:58,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:58,741 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f87f80edd94f45e1a7476fb935fdf519, entries=150, sequenceid=309, filesize=12.0 K 2024-11-23T15:42:58,742 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f77eb2e3d7d1fc2f4113a5c7f8d27655 in 1254ms, sequenceid=309, compaction requested=false 2024-11-23T15:42:58,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:58,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:58,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-23T15:42:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-23T15:42:58,744 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-23T15:42:58,744 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8660 sec 2024-11-23T15:42:58,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.8690 sec 2024-11-23T15:42:58,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:58,866 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T15:42:58,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T15:42:58,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123682fc14502cd4b71825f6d60837e37df_f77eb2e3d7d1fc2f4113a5c7f8d27655 is 50, key is test_row_0/A:col10/1732376577727/Put/seqid=0 2024-11-23T15:42:58,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742422_1598 (size=12454) 2024-11-23T15:42:58,876 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:42:58,879 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123682fc14502cd4b71825f6d60837e37df_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123682fc14502cd4b71825f6d60837e37df_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:42:58,879 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/6e835ed8a059417e8801c9b39f685826, store: [table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:58,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/6e835ed8a059417e8801c9b39f685826 is 175, key is test_row_0/A:col10/1732376577727/Put/seqid=0 2024-11-23T15:42:58,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742423_1599 (size=31255) 2024-11-23T15:42:58,884 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/6e835ed8a059417e8801c9b39f685826 2024-11-23T15:42:58,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/e7995f69e874490abfbcc7c291d6affa is 50, key is test_row_0/B:col10/1732376577727/Put/seqid=0 2024-11-23T15:42:58,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742424_1600 (size=12301) 2024-11-23T15:42:58,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:58,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376638895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376638895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:58,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:58,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376638896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376638895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:58,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:59,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:59,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376639004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:59,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:59,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:59,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376639004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:59,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:59,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:59,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376639004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:59,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376639004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:59,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-23T15:42:59,024 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:42:59,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T15:42:59,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-23T15:42:59,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-23T15:42:59,025 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T15:42:59,026 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T15:42:59,026 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T15:42:59,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-23T15:42:59,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-23T15:42:59,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:59,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:59,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:59,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:59,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:59,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:59,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376639214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:59,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:59,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376639214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376639214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:59,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376639214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:59,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:59,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/e7995f69e874490abfbcc7c291d6affa 2024-11-23T15:42:59,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/2617cb00f919414cb8cc3711d5b07114 is 50, key is test_row_0/C:col10/1732376577727/Put/seqid=0 2024-11-23T15:42:59,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742425_1601 (size=12301) 2024-11-23T15:42:59,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-23T15:42:59,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:59,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:59,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:59,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:59,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:59,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-23T15:42:59,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-23T15:42:59,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:59,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:59,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:59,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:59,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:59,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52714 deadline: 1732376639524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:59,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52746 deadline: 1732376639524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:59,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:59,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:59,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52756 deadline: 1732376639524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:59,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T15:42:59,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:59,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46589 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52732 deadline: 1732376639524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 2024-11-23T15:42:59,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 , the old value is region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T15:42:59,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., hostname=b712f9af2c12,46589,1732376409194, seqNum=5 because the exception is null or not the one we care about 2024-11-23T15:42:59,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-23T15:42:59,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:59,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:42:59,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:59,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:59,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:59,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T15:42:59,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-23T15:42:59,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/2617cb00f919414cb8cc3711d5b07114 2024-11-23T15:42:59,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/6e835ed8a059417e8801c9b39f685826 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/6e835ed8a059417e8801c9b39f685826 2024-11-23T15:42:59,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/6e835ed8a059417e8801c9b39f685826, entries=150, sequenceid=324, filesize=30.5 K 2024-11-23T15:42:59,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/e7995f69e874490abfbcc7c291d6affa as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/e7995f69e874490abfbcc7c291d6affa 2024-11-23T15:42:59,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/e7995f69e874490abfbcc7c291d6affa, entries=150, sequenceid=324, filesize=12.0 K 2024-11-23T15:42:59,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/2617cb00f919414cb8cc3711d5b07114 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2617cb00f919414cb8cc3711d5b07114 2024-11-23T15:42:59,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2617cb00f919414cb8cc3711d5b07114, entries=150, sequenceid=324, filesize=12.0 K 2024-11-23T15:42:59,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f77eb2e3d7d1fc2f4113a5c7f8d27655 in 847ms, sequenceid=324, compaction requested=true 2024-11-23T15:42:59,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:42:59,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f77eb2e3d7d1fc2f4113a5c7f8d27655:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T15:42:59,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:59,713 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:59,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f77eb2e3d7d1fc2f4113a5c7f8d27655:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T15:42:59,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:42:59,713 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:42:59,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f77eb2e3d7d1fc2f4113a5c7f8d27655:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T15:42:59,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:42:59,714 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:59,714 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94554 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:42:59,714 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): f77eb2e3d7d1fc2f4113a5c7f8d27655/B is initiating minor compaction (all files) 2024-11-23T15:42:59,714 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1541): f77eb2e3d7d1fc2f4113a5c7f8d27655/A is initiating minor compaction (all files) 2024-11-23T15:42:59,714 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f77eb2e3d7d1fc2f4113a5c7f8d27655/B in TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:59,714 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f77eb2e3d7d1fc2f4113a5c7f8d27655/A in TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:59,714 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/9cb054b5206b444eaab49c21d60d24d1, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/ce237ac8d09a4594bb5a490d16e5ff08, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/6e835ed8a059417e8801c9b39f685826] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp, totalSize=92.3 K 2024-11-23T15:42:59,714 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/52999accd2814f908601f9b26f2db7dc, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/1afb0b7577c04fdd933df1bc0b009bba, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/e7995f69e874490abfbcc7c291d6affa] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp, totalSize=36.7 K 2024-11-23T15:42:59,714 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:59,714 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. files: [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/9cb054b5206b444eaab49c21d60d24d1, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/ce237ac8d09a4594bb5a490d16e5ff08, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/6e835ed8a059417e8801c9b39f685826] 2024-11-23T15:42:59,714 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 52999accd2814f908601f9b26f2db7dc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732376574344 2024-11-23T15:42:59,714 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cb054b5206b444eaab49c21d60d24d1, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732376574344 2024-11-23T15:42:59,715 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 1afb0b7577c04fdd933df1bc0b009bba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732376576576 2024-11-23T15:42:59,715 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce237ac8d09a4594bb5a490d16e5ff08, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732376576576 2024-11-23T15:42:59,715 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting e7995f69e874490abfbcc7c291d6affa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732376577725 2024-11-23T15:42:59,715 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e835ed8a059417e8801c9b39f685826, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732376577725 2024-11-23T15:42:59,718 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:59,719 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f77eb2e3d7d1fc2f4113a5c7f8d27655#B#compaction#510 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:42:59,720 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/600c289c191b441b948f01a7222895d8 is 50, key is test_row_0/B:col10/1732376577727/Put/seqid=0 2024-11-23T15:42:59,721 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123664dd7898aeb4cff8c6ffe5d06ce0d0d_f77eb2e3d7d1fc2f4113a5c7f8d27655 store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:59,723 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123664dd7898aeb4cff8c6ffe5d06ce0d0d_f77eb2e3d7d1fc2f4113a5c7f8d27655, store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:59,723 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123664dd7898aeb4cff8c6ffe5d06ce0d0d_f77eb2e3d7d1fc2f4113a5c7f8d27655 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:42:59,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742426_1602 (size=13085) 2024-11-23T15:42:59,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742427_1603 (size=4469) 2024-11-23T15:42:59,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46589 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-23T15:42:59,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:42:59,789 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T15:42:59,792 INFO [Thread-2295 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2295. 2024-11-23T15:42:59,792 INFO [Thread-2293 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2293. 2024-11-23T15:42:59,792 DEBUG [Thread-2295 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:59,792 DEBUG [Thread-2293 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:59,792 DEBUG [Thread-2293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:59,792 DEBUG [Thread-2295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:59,792 DEBUG [Thread-2293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:59,792 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:59,792 DEBUG [Thread-2295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:59,793 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:59,793 INFO [Thread-2291 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2291. 2024-11-23T15:42:59,793 DEBUG [Thread-2291 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:59,793 DEBUG [Thread-2291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:59,793 DEBUG [Thread-2291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:59,793 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:59,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e2b6725d3d2d4f68bdcea9735a505842_f77eb2e3d7d1fc2f4113a5c7f8d27655 is 50, key is test_row_0/A:col10/1732376578895/Put/seqid=0 2024-11-23T15:42:59,795 INFO [Thread-2297 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2297. 2024-11-23T15:42:59,795 INFO [Thread-2299 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2299. 2024-11-23T15:42:59,795 DEBUG [Thread-2297 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:59,795 DEBUG [Thread-2299 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:42:59,795 DEBUG [Thread-2297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:59,795 DEBUG [Thread-2299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:59,795 DEBUG [Thread-2297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:59,795 DEBUG [Thread-2299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:42:59,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:59,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:42:59,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742428_1604 (size=12454) 2024-11-23T15:43:00,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46589 {}] regionserver.HRegion(8855): Flush requested on f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:00,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. as already flushing 2024-11-23T15:43:00,036 INFO [Thread-2284 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2284. 2024-11-23T15:43:00,036 INFO [Thread-2280 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2280. 2024-11-23T15:43:00,036 INFO [Thread-2286 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2286. 2024-11-23T15:43:00,036 DEBUG [Thread-2284 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:43:00,036 DEBUG [Thread-2286 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:43:00,036 DEBUG [Thread-2280 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:43:00,036 DEBUG [Thread-2284 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:43:00,036 DEBUG [Thread-2286 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:43:00,036 DEBUG [Thread-2280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:43:00,036 DEBUG [Thread-2284 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:43:00,036 DEBUG [Thread-2286 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:43:00,036 DEBUG [Thread-2280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:43:00,036 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:43:00,036 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:43:00,036 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:43:00,037 INFO [Thread-2288 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2288. 2024-11-23T15:43:00,037 DEBUG [Thread-2288 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:43:00,037 DEBUG [Thread-2288 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:43:00,037 DEBUG [Thread-2288 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:43:00,037 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:43:00,128 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f77eb2e3d7d1fc2f4113a5c7f8d27655#A#compaction#511 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:43:00,129 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/d947742f8d974ec29a5b284242756bee is 175, key is test_row_0/A:col10/1732376577727/Put/seqid=0 2024-11-23T15:43:00,133 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/600c289c191b441b948f01a7222895d8 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/600c289c191b441b948f01a7222895d8 2024-11-23T15:43:00,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742429_1605 (size=32039) 2024-11-23T15:43:00,139 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f77eb2e3d7d1fc2f4113a5c7f8d27655/B of f77eb2e3d7d1fc2f4113a5c7f8d27655 into 600c289c191b441b948f01a7222895d8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:43:00,139 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:43:00,140 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., storeName=f77eb2e3d7d1fc2f4113a5c7f8d27655/B, priority=13, startTime=1732376579713; duration=0sec 2024-11-23T15:43:00,140 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T15:43:00,140 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f77eb2e3d7d1fc2f4113a5c7f8d27655:B 2024-11-23T15:43:00,140 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T15:43:00,141 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T15:43:00,141 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1541): f77eb2e3d7d1fc2f4113a5c7f8d27655/C is initiating minor compaction (all files) 2024-11-23T15:43:00,141 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f77eb2e3d7d1fc2f4113a5c7f8d27655/C in TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:43:00,141 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/b56149b55323484ba4910d8d3058ec90, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f87f80edd94f45e1a7476fb935fdf519, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2617cb00f919414cb8cc3711d5b07114] into tmpdir=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp, totalSize=36.7 K 2024-11-23T15:43:00,142 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting b56149b55323484ba4910d8d3058ec90, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732376574344 2024-11-23T15:43:00,142 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting f87f80edd94f45e1a7476fb935fdf519, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732376576576 2024-11-23T15:43:00,143 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] compactions.Compactor(225): Compacting 2617cb00f919414cb8cc3711d5b07114, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732376577725 2024-11-23T15:43:00,150 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f77eb2e3d7d1fc2f4113a5c7f8d27655#C#compaction#513 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T15:43:00,150 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/a8cd76609d034d39b82c85184bbc60a8 is 50, key is test_row_0/C:col10/1732376577727/Put/seqid=0 2024-11-23T15:43:00,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742430_1606 (size=13085) 2024-11-23T15:43:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-23T15:43:00,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:43:00,205 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e2b6725d3d2d4f68bdcea9735a505842_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e2b6725d3d2d4f68bdcea9735a505842_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:00,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/765c94defd184ae5a6e695df611a6fa0, store: [table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:43:00,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/765c94defd184ae5a6e695df611a6fa0 is 175, key is test_row_0/A:col10/1732376578895/Put/seqid=0 2024-11-23T15:43:00,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742431_1607 (size=31255) 2024-11-23T15:43:00,543 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/d947742f8d974ec29a5b284242756bee as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/d947742f8d974ec29a5b284242756bee 2024-11-23T15:43:00,549 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f77eb2e3d7d1fc2f4113a5c7f8d27655/A of f77eb2e3d7d1fc2f4113a5c7f8d27655 into d947742f8d974ec29a5b284242756bee(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:43:00,549 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:43:00,549 INFO [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., storeName=f77eb2e3d7d1fc2f4113a5c7f8d27655/A, priority=13, startTime=1732376579713; duration=0sec 2024-11-23T15:43:00,549 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:43:00,549 DEBUG [RS:0;b712f9af2c12:46589-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f77eb2e3d7d1fc2f4113a5c7f8d27655:A 2024-11-23T15:43:00,558 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/a8cd76609d034d39b82c85184bbc60a8 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/a8cd76609d034d39b82c85184bbc60a8 2024-11-23T15:43:00,563 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f77eb2e3d7d1fc2f4113a5c7f8d27655/C of f77eb2e3d7d1fc2f4113a5c7f8d27655 into a8cd76609d034d39b82c85184bbc60a8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T15:43:00,563 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:43:00,563 INFO [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., storeName=f77eb2e3d7d1fc2f4113a5c7f8d27655/C, priority=13, startTime=1732376579713; duration=0sec 2024-11-23T15:43:00,563 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T15:43:00,563 DEBUG [RS:0;b712f9af2c12:46589-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f77eb2e3d7d1fc2f4113a5c7f8d27655:C 2024-11-23T15:43:00,614 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=348, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/765c94defd184ae5a6e695df611a6fa0 2024-11-23T15:43:00,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/66ed811ec7ae4155bbd2ad3d753b8a70 is 50, key is test_row_0/B:col10/1732376578895/Put/seqid=0 2024-11-23T15:43:00,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742432_1608 (size=12301) 2024-11-23T15:43:00,769 INFO [Thread-2282 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2282. 2024-11-23T15:43:00,769 DEBUG [Thread-2282 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T15:43:00,769 DEBUG [Thread-2282 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:43:00,769 DEBUG [Thread-2282 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:43:00,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:43:01,030 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/66ed811ec7ae4155bbd2ad3d753b8a70 2024-11-23T15:43:01,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/58e8211483674c0b8ffdd807b77fc090 is 50, key is test_row_0/C:col10/1732376578895/Put/seqid=0 2024-11-23T15:43:01,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742433_1609 (size=12301) 2024-11-23T15:43:01,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-23T15:43:01,449 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/58e8211483674c0b8ffdd807b77fc090 2024-11-23T15:43:01,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/765c94defd184ae5a6e695df611a6fa0 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/765c94defd184ae5a6e695df611a6fa0 2024-11-23T15:43:01,462 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/765c94defd184ae5a6e695df611a6fa0, entries=150, sequenceid=348, filesize=30.5 K 2024-11-23T15:43:01,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/66ed811ec7ae4155bbd2ad3d753b8a70 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/66ed811ec7ae4155bbd2ad3d753b8a70 2024-11-23T15:43:01,467 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/66ed811ec7ae4155bbd2ad3d753b8a70, entries=150, sequenceid=348, filesize=12.0 K 2024-11-23T15:43:01,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/58e8211483674c0b8ffdd807b77fc090 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/58e8211483674c0b8ffdd807b77fc090 2024-11-23T15:43:01,470 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/58e8211483674c0b8ffdd807b77fc090, entries=150, sequenceid=348, filesize=12.0 K 2024-11-23T15:43:01,471 INFO [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=33.54 KB/34350 for f77eb2e3d7d1fc2f4113a5c7f8d27655 in 1682ms, sequenceid=348, compaction requested=false 2024-11-23T15:43:01,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: 2024-11-23T15:43:01,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:43:01,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b712f9af2c12:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-23T15:43:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-23T15:43:01,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-23T15:43:01,474 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4460 sec 2024-11-23T15:43:01,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 2.4510 sec 2024-11-23T15:43:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-23T15:43:03,196 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:43:03,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T15:43:03,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-23T15:43:03,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-23T15:43:03,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-23T15:43:03,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-23T15:43:03,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-23T15:43:03,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T15:43:03,196 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3884 2024-11-23T15:43:03,197 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3900 2024-11-23T15:43:03,197 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3798 2024-11-23T15:43:03,197 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3888 2024-11-23T15:43:03,197 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3889 2024-11-23T15:43:03,197 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T15:43:03,197 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T15:43:03,197 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T15:43:03,197 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:43:03,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:43:03,198 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:43:03,199 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-23T15:43:03,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T15:43:03,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-23T15:43:03,204 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376583204"}]},"ts":"1732376583204"} 2024-11-23T15:43:03,206 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T15:43:03,206 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T15:43:03,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T15:43:03,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=176, ppid=175, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f77eb2e3d7d1fc2f4113a5c7f8d27655, UNASSIGN}] 2024-11-23T15:43:03,210 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=176, ppid=175, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f77eb2e3d7d1fc2f4113a5c7f8d27655, UNASSIGN 2024-11-23T15:43:03,211 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=176 updating hbase:meta row=f77eb2e3d7d1fc2f4113a5c7f8d27655, regionState=CLOSING, regionLocation=b712f9af2c12,46589,1732376409194 2024-11-23T15:43:03,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=176, ppid=175, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f77eb2e3d7d1fc2f4113a5c7f8d27655, UNASSIGN because future has completed 2024-11-23T15:43:03,212 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T15:43:03,212 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE, hasLock=false; CloseRegionProcedure f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194}] 2024-11-23T15:43:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-23T15:43:03,365 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] handler.UnassignRegionHandler(122): Close f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:03,366 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T15:43:03,366 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1722): Closing f77eb2e3d7d1fc2f4113a5c7f8d27655, disabling compactions & flushes 2024-11-23T15:43:03,366 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:43:03,366 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:43:03,366 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. after waiting 0 ms 2024-11-23T15:43:03,366 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:43:03,366 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(2902): Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-23T15:43:03,378 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112379f9e8409a8148ce99a4eb1aed7e2358_f77eb2e3d7d1fc2f4113a5c7f8d27655 is 50, key is test_row_0/A:col10/1732376580036/Put/seqid=0 2024-11-23T15:43:03,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742434_1610 (size=12454) 2024-11-23T15:43:03,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-23T15:43:03,783 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T15:43:03,792 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112379f9e8409a8148ce99a4eb1aed7e2358_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112379f9e8409a8148ce99a4eb1aed7e2358_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:03,794 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/e80309f0f80241c5b50490a2ef8c9ae5, store: [table=TestAcidGuarantees family=A region=f77eb2e3d7d1fc2f4113a5c7f8d27655] 2024-11-23T15:43:03,795 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/e80309f0f80241c5b50490a2ef8c9ae5 is 175, key is test_row_0/A:col10/1732376580036/Put/seqid=0 2024-11-23T15:43:03,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742435_1611 (size=31255) 2024-11-23T15:43:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-23T15:43:04,201 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/e80309f0f80241c5b50490a2ef8c9ae5 2024-11-23T15:43:04,214 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/cb045349cfde4e8cbf149ad217dfd9ba is 50, key is test_row_0/B:col10/1732376580036/Put/seqid=0 2024-11-23T15:43:04,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742436_1612 (size=12301) 2024-11-23T15:43:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-23T15:43:04,619 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/cb045349cfde4e8cbf149ad217dfd9ba 2024-11-23T15:43:04,632 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/470527f44074458ead6c5207ff085fab is 50, key is test_row_0/C:col10/1732376580036/Put/seqid=0 2024-11-23T15:43:04,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742437_1613 (size=12301) 2024-11-23T15:43:05,037 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/470527f44074458ead6c5207ff085fab 2024-11-23T15:43:05,044 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/A/e80309f0f80241c5b50490a2ef8c9ae5 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/e80309f0f80241c5b50490a2ef8c9ae5 2024-11-23T15:43:05,047 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/e80309f0f80241c5b50490a2ef8c9ae5, entries=150, sequenceid=359, filesize=30.5 K 2024-11-23T15:43:05,048 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/B/cb045349cfde4e8cbf149ad217dfd9ba as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/cb045349cfde4e8cbf149ad217dfd9ba 2024-11-23T15:43:05,051 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/cb045349cfde4e8cbf149ad217dfd9ba, entries=150, sequenceid=359, filesize=12.0 K 2024-11-23T15:43:05,051 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/.tmp/C/470527f44074458ead6c5207ff085fab as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/470527f44074458ead6c5207ff085fab 2024-11-23T15:43:05,054 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/470527f44074458ead6c5207ff085fab, entries=150, sequenceid=359, filesize=12.0 K 2024-11-23T15:43:05,055 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for f77eb2e3d7d1fc2f4113a5c7f8d27655 in 1689ms, sequenceid=359, compaction requested=true 2024-11-23T15:43:05,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/4dd962ba33ac42439fa1635f9ad2594a, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/4e2f7ca2de3545bb9ff6aa735449cda2, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/688199b0e1114e879cd3babc2377c316, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/974c6f4b4a6c4f92a8a818dc020b2e75, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/d8ee4600e5624218bf8aa45c1be4ed99, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/9564d9e22e9d471c85ee6338fb317bc9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/a32434d423cb454db827a722372c4ef7, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/b7b09bec34074f00b6843f2e3f49e260, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/1453e986da1b4f99bc5ee88d6cfbbdad, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/57949d88ab9b46579286a274916ef3f0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/8bae3fbee2e84ef5b836f84bc0dbf9c8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/72fa4d54e6b041999fed7108b73ab8e2, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/792d122d7caa465a8a3b3e3ff166e795, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/413eff813c8448f3b171a92f9205f317, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/36986cfb216b41cf83f76e3b157d6c88, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/529cbd6eb46c4f4b9c0ee054813ead1e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/27f0af9ded4e4972b84de9186943e773, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/6951b51490fd465f97b1c7be6782c809, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/0c80e750f24d433cb0151d75f410b9a8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/d2e76ee658da4472898c22d31645fc52, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/9cb054b5206b444eaab49c21d60d24d1, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/ce237ac8d09a4594bb5a490d16e5ff08, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/6e835ed8a059417e8801c9b39f685826] to archive 2024-11-23T15:43:05,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T15:43:05,057 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/4dd962ba33ac42439fa1635f9ad2594a to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/4dd962ba33ac42439fa1635f9ad2594a 2024-11-23T15:43:05,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/4e2f7ca2de3545bb9ff6aa735449cda2 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/4e2f7ca2de3545bb9ff6aa735449cda2 2024-11-23T15:43:05,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/688199b0e1114e879cd3babc2377c316 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/688199b0e1114e879cd3babc2377c316 2024-11-23T15:43:05,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/974c6f4b4a6c4f92a8a818dc020b2e75 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/974c6f4b4a6c4f92a8a818dc020b2e75 2024-11-23T15:43:05,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/d8ee4600e5624218bf8aa45c1be4ed99 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/d8ee4600e5624218bf8aa45c1be4ed99 2024-11-23T15:43:05,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/9564d9e22e9d471c85ee6338fb317bc9 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/9564d9e22e9d471c85ee6338fb317bc9 2024-11-23T15:43:05,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/a32434d423cb454db827a722372c4ef7 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/a32434d423cb454db827a722372c4ef7 2024-11-23T15:43:05,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/b7b09bec34074f00b6843f2e3f49e260 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/b7b09bec34074f00b6843f2e3f49e260 2024-11-23T15:43:05,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/1453e986da1b4f99bc5ee88d6cfbbdad to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/1453e986da1b4f99bc5ee88d6cfbbdad 2024-11-23T15:43:05,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/57949d88ab9b46579286a274916ef3f0 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/57949d88ab9b46579286a274916ef3f0 2024-11-23T15:43:05,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/8bae3fbee2e84ef5b836f84bc0dbf9c8 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/8bae3fbee2e84ef5b836f84bc0dbf9c8 2024-11-23T15:43:05,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/72fa4d54e6b041999fed7108b73ab8e2 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/72fa4d54e6b041999fed7108b73ab8e2 2024-11-23T15:43:05,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/792d122d7caa465a8a3b3e3ff166e795 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/792d122d7caa465a8a3b3e3ff166e795 2024-11-23T15:43:05,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/413eff813c8448f3b171a92f9205f317 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/413eff813c8448f3b171a92f9205f317 2024-11-23T15:43:05,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/36986cfb216b41cf83f76e3b157d6c88 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/36986cfb216b41cf83f76e3b157d6c88 2024-11-23T15:43:05,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/529cbd6eb46c4f4b9c0ee054813ead1e to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/529cbd6eb46c4f4b9c0ee054813ead1e 2024-11-23T15:43:05,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/27f0af9ded4e4972b84de9186943e773 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/27f0af9ded4e4972b84de9186943e773 2024-11-23T15:43:05,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/6951b51490fd465f97b1c7be6782c809 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/6951b51490fd465f97b1c7be6782c809 2024-11-23T15:43:05,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/0c80e750f24d433cb0151d75f410b9a8 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/0c80e750f24d433cb0151d75f410b9a8 2024-11-23T15:43:05,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/d2e76ee658da4472898c22d31645fc52 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/d2e76ee658da4472898c22d31645fc52 2024-11-23T15:43:05,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/9cb054b5206b444eaab49c21d60d24d1 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/9cb054b5206b444eaab49c21d60d24d1 2024-11-23T15:43:05,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/ce237ac8d09a4594bb5a490d16e5ff08 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/ce237ac8d09a4594bb5a490d16e5ff08 2024-11-23T15:43:05,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/6e835ed8a059417e8801c9b39f685826 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/6e835ed8a059417e8801c9b39f685826 2024-11-23T15:43:05,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/e82f676f867a479aa5f253239be6df49, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/35402dbc187b45f9816913fc33e3e5ea, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/971fa387c67c423481ea6030ed11aa46, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/05e46f9a2943490387d88644c49db8db, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/d722014b5c894364bf2a1052e148e749, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/18b00f44939347f49efc1aae2b3264f7, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/05ef5fc3384e463ea9df9bcd5aeacaa0, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/b0bbdc753d154b2fbc3e45aac4575851, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/4a21121ef4044b398883e94323f46c7b, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/3fb5e308c2e646d989c251c46d1b9876, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/6d908cd618bf4517b567a89e82f6e5e7, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/101a4b0a9dfe451b94cf3c720ada4b92, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/a65bc193cf434df09d4f202bdc65c0a9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/a67b8c040a454dcc9173bc6790486d0e, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/9d1575edbbc146d1a51554077b952666, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/a69f35ea3ae44389af13a96a0b27f86d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/0d0fc429b81f40b3baf293aeff678134, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/06a38dedf25e4f4e99c647a6e2948186, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/e55882e55e7d4fc688f82a492da92389, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/52999accd2814f908601f9b26f2db7dc, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/d0944a7b951742c687c34ce22e041595, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/1afb0b7577c04fdd933df1bc0b009bba, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/e7995f69e874490abfbcc7c291d6affa] to archive 2024-11-23T15:43:05,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T15:43:05,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/e82f676f867a479aa5f253239be6df49 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/e82f676f867a479aa5f253239be6df49 2024-11-23T15:43:05,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/35402dbc187b45f9816913fc33e3e5ea to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/35402dbc187b45f9816913fc33e3e5ea 2024-11-23T15:43:05,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/971fa387c67c423481ea6030ed11aa46 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/971fa387c67c423481ea6030ed11aa46 2024-11-23T15:43:05,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/05e46f9a2943490387d88644c49db8db to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/05e46f9a2943490387d88644c49db8db 2024-11-23T15:43:05,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/d722014b5c894364bf2a1052e148e749 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/d722014b5c894364bf2a1052e148e749 2024-11-23T15:43:05,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/18b00f44939347f49efc1aae2b3264f7 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/18b00f44939347f49efc1aae2b3264f7 2024-11-23T15:43:05,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/05ef5fc3384e463ea9df9bcd5aeacaa0 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/05ef5fc3384e463ea9df9bcd5aeacaa0 2024-11-23T15:43:05,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/b0bbdc753d154b2fbc3e45aac4575851 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/b0bbdc753d154b2fbc3e45aac4575851 2024-11-23T15:43:05,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/4a21121ef4044b398883e94323f46c7b to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/4a21121ef4044b398883e94323f46c7b 2024-11-23T15:43:05,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/3fb5e308c2e646d989c251c46d1b9876 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/3fb5e308c2e646d989c251c46d1b9876 2024-11-23T15:43:05,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/6d908cd618bf4517b567a89e82f6e5e7 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/6d908cd618bf4517b567a89e82f6e5e7 2024-11-23T15:43:05,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/101a4b0a9dfe451b94cf3c720ada4b92 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/101a4b0a9dfe451b94cf3c720ada4b92 2024-11-23T15:43:05,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/a65bc193cf434df09d4f202bdc65c0a9 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/a65bc193cf434df09d4f202bdc65c0a9 2024-11-23T15:43:05,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/a67b8c040a454dcc9173bc6790486d0e to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/a67b8c040a454dcc9173bc6790486d0e 2024-11-23T15:43:05,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/9d1575edbbc146d1a51554077b952666 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/9d1575edbbc146d1a51554077b952666 2024-11-23T15:43:05,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/a69f35ea3ae44389af13a96a0b27f86d to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/a69f35ea3ae44389af13a96a0b27f86d 2024-11-23T15:43:05,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/0d0fc429b81f40b3baf293aeff678134 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/0d0fc429b81f40b3baf293aeff678134 2024-11-23T15:43:05,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/06a38dedf25e4f4e99c647a6e2948186 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/06a38dedf25e4f4e99c647a6e2948186 2024-11-23T15:43:05,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/e55882e55e7d4fc688f82a492da92389 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/e55882e55e7d4fc688f82a492da92389 2024-11-23T15:43:05,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/52999accd2814f908601f9b26f2db7dc to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/52999accd2814f908601f9b26f2db7dc 2024-11-23T15:43:05,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/d0944a7b951742c687c34ce22e041595 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/d0944a7b951742c687c34ce22e041595 2024-11-23T15:43:05,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/1afb0b7577c04fdd933df1bc0b009bba to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/1afb0b7577c04fdd933df1bc0b009bba 2024-11-23T15:43:05,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/e7995f69e874490abfbcc7c291d6affa to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/e7995f69e874490abfbcc7c291d6affa 2024-11-23T15:43:05,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2dd734aa745441c780edd3eea9b76b91, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/9211eb22c49447d5bf17a0cd0c1a8350, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/126cfb273c5a45ec8865819d8a1e4908, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/0460740ae86b426b98c98ca90a323239, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/47eb1413f0c3430da822489cc8f51cc6, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f7e0ebf6b26342ad941ca911828427a3, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/cacf24dafd584064bbcf3cfdff328452, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/65f8d0d38b464c5f8464edd7c4d8c8c8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2c9f74938975405f86ba8aa9bea470bf, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f364ca57f8934091a37121233ce5e76f, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2b4b411417b545f3b2aedef284d667d8, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/922cb22a07434c91b02209335350a94c, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/8eef402c7ea74980ae6c4f49c615c2f5, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/8b0bf8092c31463e9ebf5cd54aac933d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/d5b99dfc4107427a84dd2ec5bcc6afe9, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2bd0352a88cc4391826f044f1694806d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/a6ecfbe8f8f5401a9d312618835117df, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f5d5ca7800c74816a46f99d70702afac, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/245b13c41e204575aeb48a01de90328d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/b56149b55323484ba4910d8d3058ec90, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/07f28809e3b84e8d84921ad3d831c20d, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f87f80edd94f45e1a7476fb935fdf519, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2617cb00f919414cb8cc3711d5b07114] to archive 2024-11-23T15:43:05,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T15:43:05,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2dd734aa745441c780edd3eea9b76b91 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2dd734aa745441c780edd3eea9b76b91 2024-11-23T15:43:05,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/9211eb22c49447d5bf17a0cd0c1a8350 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/9211eb22c49447d5bf17a0cd0c1a8350 2024-11-23T15:43:05,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/126cfb273c5a45ec8865819d8a1e4908 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/126cfb273c5a45ec8865819d8a1e4908 2024-11-23T15:43:05,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/0460740ae86b426b98c98ca90a323239 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/0460740ae86b426b98c98ca90a323239 2024-11-23T15:43:05,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/47eb1413f0c3430da822489cc8f51cc6 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/47eb1413f0c3430da822489cc8f51cc6 2024-11-23T15:43:05,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f7e0ebf6b26342ad941ca911828427a3 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f7e0ebf6b26342ad941ca911828427a3 2024-11-23T15:43:05,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/cacf24dafd584064bbcf3cfdff328452 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/cacf24dafd584064bbcf3cfdff328452 2024-11-23T15:43:05,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/65f8d0d38b464c5f8464edd7c4d8c8c8 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/65f8d0d38b464c5f8464edd7c4d8c8c8 2024-11-23T15:43:05,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2c9f74938975405f86ba8aa9bea470bf to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2c9f74938975405f86ba8aa9bea470bf 2024-11-23T15:43:05,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f364ca57f8934091a37121233ce5e76f to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f364ca57f8934091a37121233ce5e76f 2024-11-23T15:43:05,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2b4b411417b545f3b2aedef284d667d8 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2b4b411417b545f3b2aedef284d667d8 2024-11-23T15:43:05,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/922cb22a07434c91b02209335350a94c to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/922cb22a07434c91b02209335350a94c 2024-11-23T15:43:05,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/8eef402c7ea74980ae6c4f49c615c2f5 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/8eef402c7ea74980ae6c4f49c615c2f5 2024-11-23T15:43:05,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/8b0bf8092c31463e9ebf5cd54aac933d to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/8b0bf8092c31463e9ebf5cd54aac933d 2024-11-23T15:43:05,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/d5b99dfc4107427a84dd2ec5bcc6afe9 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/d5b99dfc4107427a84dd2ec5bcc6afe9 2024-11-23T15:43:05,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2bd0352a88cc4391826f044f1694806d to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2bd0352a88cc4391826f044f1694806d 2024-11-23T15:43:05,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/a6ecfbe8f8f5401a9d312618835117df to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/a6ecfbe8f8f5401a9d312618835117df 2024-11-23T15:43:05,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f5d5ca7800c74816a46f99d70702afac to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f5d5ca7800c74816a46f99d70702afac 2024-11-23T15:43:05,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/245b13c41e204575aeb48a01de90328d to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/245b13c41e204575aeb48a01de90328d 2024-11-23T15:43:05,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/b56149b55323484ba4910d8d3058ec90 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/b56149b55323484ba4910d8d3058ec90 2024-11-23T15:43:05,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/07f28809e3b84e8d84921ad3d831c20d to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/07f28809e3b84e8d84921ad3d831c20d 2024-11-23T15:43:05,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f87f80edd94f45e1a7476fb935fdf519 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/f87f80edd94f45e1a7476fb935fdf519 2024-11-23T15:43:05,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2617cb00f919414cb8cc3711d5b07114 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/2617cb00f919414cb8cc3711d5b07114 2024-11-23T15:43:05,108 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/recovered.edits/362.seqid, newMaxSeqId=362, maxSeqId=4 2024-11-23T15:43:05,108 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. 2024-11-23T15:43:05,108 DEBUG [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] regionserver.HRegion(1676): Region close journal for f77eb2e3d7d1fc2f4113a5c7f8d27655: Waiting for close lock at 1732376583366Running coprocessor pre-close hooks at 1732376583366Disabling compacts and flushes for region at 1732376583366Disabling writes for close at 1732376583366Obtaining lock to block concurrent updates at 1732376583367 (+1 ms)Preparing flush snapshotting stores in f77eb2e3d7d1fc2f4113a5c7f8d27655 at 1732376583367Finished memstore snapshotting TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732376583367Flushing stores of TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655. at 1732376583369 (+2 ms)Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655/A: creating writer at 1732376583369Flushing mob file f77eb2e3d7d1fc2f4113a5c7f8d27655/A: appending metadata at 1732376583377 (+8 ms)Flushing mob file f77eb2e3d7d1fc2f4113a5c7f8d27655/A: closing flushed file at 1732376583377Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655/A: appending metadata at 1732376583794 (+417 ms)Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655/A: closing flushed file at 1732376583794Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655/B: creating writer at 1732376584210 (+416 ms)Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655/B: appending metadata at 1732376584213 (+3 ms)Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655/B: closing flushed file at 1732376584213Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655/C: creating writer at 1732376584628 (+415 ms)Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655/C: appending metadata at 1732376584632 (+4 ms)Flushing f77eb2e3d7d1fc2f4113a5c7f8d27655/C: closing flushed file at 1732376584632Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55e1f4e8: reopening flushed file at 1732376585043 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62675f6b: reopening flushed file at 1732376585047 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ecbf97f: reopening flushed file at 1732376585051 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for f77eb2e3d7d1fc2f4113a5c7f8d27655 in 1689ms, sequenceid=359, compaction requested=true at 1732376585055 (+4 ms)Writing region close event to WAL at 1732376585106 (+51 ms)Running coprocessor post-close hooks at 1732376585108 (+2 ms)Closed at 1732376585108 2024-11-23T15:43:05,110 INFO [RS_CLOSE_REGION-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_REGION, pid=177}] handler.UnassignRegionHandler(157): Closed f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:05,110 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=176 updating hbase:meta row=f77eb2e3d7d1fc2f4113a5c7f8d27655, regionState=CLOSED 2024-11-23T15:43:05,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=177, ppid=176, state=RUNNABLE, hasLock=false; CloseRegionProcedure f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 because future has completed 2024-11-23T15:43:05,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-23T15:43:05,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; CloseRegionProcedure f77eb2e3d7d1fc2f4113a5c7f8d27655, server=b712f9af2c12,46589,1732376409194 in 1.9000 sec 2024-11-23T15:43:05,114 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=176, resume processing ppid=175 2024-11-23T15:43:05,114 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, ppid=175, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f77eb2e3d7d1fc2f4113a5c7f8d27655, UNASSIGN in 1.9040 sec 2024-11-23T15:43:05,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-23T15:43:05,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9070 sec 2024-11-23T15:43:05,117 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732376585117"}]},"ts":"1732376585117"} 2024-11-23T15:43:05,164 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T15:43:05,164 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T15:43:05,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9650 sec 2024-11-23T15:43:05,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-23T15:43:05,355 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:43:05,356 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-23T15:43:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:43:05,359 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=178, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:43:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-23T15:43:05,360 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=178, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:43:05,365 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:05,369 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A, FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B, FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C, FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/recovered.edits] 2024-11-23T15:43:05,374 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/765c94defd184ae5a6e695df611a6fa0 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/765c94defd184ae5a6e695df611a6fa0 2024-11-23T15:43:05,376 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/d947742f8d974ec29a5b284242756bee to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/d947742f8d974ec29a5b284242756bee 2024-11-23T15:43:05,378 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/e80309f0f80241c5b50490a2ef8c9ae5 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/A/e80309f0f80241c5b50490a2ef8c9ae5 2024-11-23T15:43:05,381 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/600c289c191b441b948f01a7222895d8 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/600c289c191b441b948f01a7222895d8 2024-11-23T15:43:05,383 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/66ed811ec7ae4155bbd2ad3d753b8a70 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/66ed811ec7ae4155bbd2ad3d753b8a70 2024-11-23T15:43:05,385 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/cb045349cfde4e8cbf149ad217dfd9ba to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/B/cb045349cfde4e8cbf149ad217dfd9ba 2024-11-23T15:43:05,388 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/470527f44074458ead6c5207ff085fab to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/470527f44074458ead6c5207ff085fab 2024-11-23T15:43:05,389 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/58e8211483674c0b8ffdd807b77fc090 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/58e8211483674c0b8ffdd807b77fc090 2024-11-23T15:43:05,391 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/a8cd76609d034d39b82c85184bbc60a8 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/C/a8cd76609d034d39b82c85184bbc60a8 2024-11-23T15:43:05,395 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/recovered.edits/362.seqid to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655/recovered.edits/362.seqid 2024-11-23T15:43:05,395 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/default/TestAcidGuarantees/f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:05,396 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T15:43:05,396 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T15:43:05,397 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-23T15:43:05,402 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112307403055b35847ad86c9c0a99610f488_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112307403055b35847ad86c9c0a99610f488_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:05,405 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230d8a0c1155b440708712674a40d3a718_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230d8a0c1155b440708712674a40d3a718_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:05,407 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231441f9ad06f442278fbe440171cc45c4_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231441f9ad06f442278fbe440171cc45c4_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:05,409 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112326fdbe2675de4817bfc1558b9c67260e_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112326fdbe2675de4817bfc1558b9c67260e_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:05,410 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112352bebd9b321f49588d77b92a03dace20_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112352bebd9b321f49588d77b92a03dace20_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:05,411 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112355a49ca3fc7a40bbba37d69a0d2ea2ca_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112355a49ca3fc7a40bbba37d69a0d2ea2ca_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:05,412 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123575a1c7c6d1f4da7968622b53b43a07c_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123575a1c7c6d1f4da7968622b53b43a07c_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:05,413 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123577a72a415d64ec1aa946abfe3337d8a_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123577a72a415d64ec1aa946abfe3337d8a_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:05,414 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235dc829bd57d94b05b3f942803149a63a_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235dc829bd57d94b05b3f942803149a63a_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:05,415 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123682fc14502cd4b71825f6d60837e37df_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123682fc14502cd4b71825f6d60837e37df_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:05,416 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112379f9e8409a8148ce99a4eb1aed7e2358_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112379f9e8409a8148ce99a4eb1aed7e2358_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:05,416 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112380d9346a18f4455080da5d7b0216d275_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112380d9346a18f4455080da5d7b0216d275_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:05,417 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123a92aad3f198f41afa6280c99072019dc_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123a92aad3f198f41afa6280c99072019dc_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:05,418 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c862458a8dd74859be04e4a13e159343_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c862458a8dd74859be04e4a13e159343_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:05,419 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ccd5594573fb435789a1247897a5adc2_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ccd5594573fb435789a1247897a5adc2_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:05,419 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123dc8aa8f79078452d8565e428233e036d_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123dc8aa8f79078452d8565e428233e036d_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:05,420 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e2b6725d3d2d4f68bdcea9735a505842_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e2b6725d3d2d4f68bdcea9735a505842_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:05,422 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ea35209dacc741fe84a1911726c97542_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ea35209dacc741fe84a1911726c97542_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:05,423 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123eede40c34c5149e0bc84ca2f22212386_f77eb2e3d7d1fc2f4113a5c7f8d27655 to hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123eede40c34c5149e0bc84ca2f22212386_f77eb2e3d7d1fc2f4113a5c7f8d27655 2024-11-23T15:43:05,423 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T15:43:05,425 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=178, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:43:05,427 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T15:43:05,430 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T15:43:05,431 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=178, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:43:05,431 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T15:43:05,431 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732376585431"}]},"ts":"9223372036854775807"} 2024-11-23T15:43:05,433 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T15:43:05,433 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f77eb2e3d7d1fc2f4113a5c7f8d27655, NAME => 'TestAcidGuarantees,,1732376556640.f77eb2e3d7d1fc2f4113a5c7f8d27655.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T15:43:05,433 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T15:43:05,433 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732376585433"}]},"ts":"9223372036854775807"} 2024-11-23T15:43:05,435 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T15:43:05,436 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=178, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T15:43:05,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 80 msec 2024-11-23T15:43:05,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42351 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-23T15:43:05,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T15:43:05,464 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T15:43:05,473 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=242 (was 239) - Thread LEAK? -, OpenFileDescriptor=455 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=366 (was 375), ProcessCount=11 (was 11), AvailableMemoryMB=7157 (was 7405) 2024-11-23T15:43:05,473 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T15:43:05,473 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T15:43:05,473 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T15:43:05,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:43:05,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:43:05,473 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T15:43:05,473 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T15:43:05,473 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=841754823, stopped=false 2024-11-23T15:43:05,474 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=b712f9af2c12,42351,1732376408377 2024-11-23T15:43:05,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46589-0x10169a3c09b0001, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T15:43:05,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T15:43:05,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46589-0x10169a3c09b0001, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:43:05,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:43:05,503 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T15:43:05,504 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46589-0x10169a3c09b0001, quorum=127.0.0.1:58385, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T15:43:05,505 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T15:43:05,505 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T15:43:05,505 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T15:43:05,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:43:05,506 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b712f9af2c12,46589,1732376409194' ***** 2024-11-23T15:43:05,506 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T15:43:05,506 INFO [RS:0;b712f9af2c12:46589 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T15:43:05,507 INFO [RS:0;b712f9af2c12:46589 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T15:43:05,507 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T15:43:05,507 INFO [RS:0;b712f9af2c12:46589 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T15:43:05,507 INFO [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(959): stopping server b712f9af2c12,46589,1732376409194 2024-11-23T15:43:05,508 INFO [RS:0;b712f9af2c12:46589 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T15:43:05,508 INFO [RS:0;b712f9af2c12:46589 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b712f9af2c12:46589. 2024-11-23T15:43:05,508 DEBUG [RS:0;b712f9af2c12:46589 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T15:43:05,508 DEBUG [RS:0;b712f9af2c12:46589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:43:05,508 INFO [RS:0;b712f9af2c12:46589 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T15:43:05,508 INFO [RS:0;b712f9af2c12:46589 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T15:43:05,508 INFO [RS:0;b712f9af2c12:46589 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T15:43:05,508 INFO [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T15:43:05,509 INFO [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-23T15:43:05,509 DEBUG [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-23T15:43:05,510 DEBUG [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T15:43:05,510 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T15:43:05,510 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T15:43:05,510 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T15:43:05,510 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T15:43:05,510 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T15:43:05,510 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-23T15:43:05,531 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740/.tmp/info/6b77172c703f4f60b66888f5f60854e0 is 95, key is TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0./info:/1732376444841/DeleteFamily/seqid=0 2024-11-23T15:43:05,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742438_1614 (size=6514) 2024-11-23T15:43:05,560 INFO [regionserver/b712f9af2c12:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T15:43:05,710 DEBUG [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T15:43:05,910 DEBUG [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T15:43:05,935 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740/.tmp/info/6b77172c703f4f60b66888f5f60854e0 2024-11-23T15:43:05,960 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740/.tmp/ns/0e117eaecdd44cbea02eb03ddcee2b65 is 93, key is TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0./ns:/1732376444801/DeleteFamily/seqid=0 2024-11-23T15:43:05,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742439_1615 (size=5906) 2024-11-23T15:43:06,111 DEBUG [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T15:43:06,155 INFO [regionserver/b712f9af2c12:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-23T15:43:06,156 INFO [regionserver/b712f9af2c12:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-23T15:43:06,311 DEBUG [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T15:43:06,365 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740/.tmp/ns/0e117eaecdd44cbea02eb03ddcee2b65 2024-11-23T15:43:06,392 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740/.tmp/rep_barrier/cadb606f53a74cdf9f07a858be06c486 is 102, key is TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0./rep_barrier:/1732376444801/DeleteFamily/seqid=0 2024-11-23T15:43:06,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742440_1616 (size=6025) 2024-11-23T15:43:06,511 DEBUG [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T15:43:06,711 DEBUG [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T15:43:06,796 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740/.tmp/rep_barrier/cadb606f53a74cdf9f07a858be06c486 2024-11-23T15:43:06,824 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740/.tmp/table/1dbb140d3ac5431795904ebd54d2deed is 96, key is TestAcidGuarantees,,1732376413849.dad1202300d1ddd3a95c32af1ad98ed0./table:/1732376444801/DeleteFamily/seqid=0 2024-11-23T15:43:06,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742441_1617 (size=5936) 2024-11-23T15:43:06,912 DEBUG [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T15:43:07,112 DEBUG [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T15:43:07,229 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740/.tmp/table/1dbb140d3ac5431795904ebd54d2deed 2024-11-23T15:43:07,238 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740/.tmp/info/6b77172c703f4f60b66888f5f60854e0 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740/info/6b77172c703f4f60b66888f5f60854e0 2024-11-23T15:43:07,243 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740/info/6b77172c703f4f60b66888f5f60854e0, entries=12, sequenceid=90, filesize=6.4 K 2024-11-23T15:43:07,244 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740/.tmp/ns/0e117eaecdd44cbea02eb03ddcee2b65 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740/ns/0e117eaecdd44cbea02eb03ddcee2b65 2024-11-23T15:43:07,248 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740/ns/0e117eaecdd44cbea02eb03ddcee2b65, entries=8, sequenceid=90, filesize=5.8 K 2024-11-23T15:43:07,248 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740/.tmp/rep_barrier/cadb606f53a74cdf9f07a858be06c486 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740/rep_barrier/cadb606f53a74cdf9f07a858be06c486 2024-11-23T15:43:07,252 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740/rep_barrier/cadb606f53a74cdf9f07a858be06c486, entries=6, sequenceid=90, filesize=5.9 K 2024-11-23T15:43:07,252 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740/.tmp/table/1dbb140d3ac5431795904ebd54d2deed as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740/table/1dbb140d3ac5431795904ebd54d2deed 2024-11-23T15:43:07,256 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740/table/1dbb140d3ac5431795904ebd54d2deed, entries=7, sequenceid=90, filesize=5.8 K 2024-11-23T15:43:07,257 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1747ms, sequenceid=90, compaction requested=false 2024-11-23T15:43:07,261 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-23T15:43:07,261 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T15:43:07,261 INFO [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T15:43:07,261 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732376585509Running coprocessor pre-close hooks at 1732376585510 (+1 ms)Disabling compacts and flushes for region at 1732376585510Disabling writes for close at 1732376585510Obtaining lock to block concurrent updates at 1732376585510Preparing flush snapshotting stores in 1588230740 at 1732376585510Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732376585511 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732376585511Flushing 1588230740/info: creating writer at 1732376585512 (+1 ms)Flushing 1588230740/info: appending metadata at 1732376585531 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732376585531Flushing 1588230740/ns: creating writer at 1732376585943 (+412 ms)Flushing 1588230740/ns: appending metadata at 1732376585960 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732376585960Flushing 1588230740/rep_barrier: creating writer at 1732376586374 (+414 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732376586391 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732376586391Flushing 1588230740/table: creating writer at 1732376586807 (+416 ms)Flushing 1588230740/table: appending metadata at 1732376586824 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732376586824Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ad38fd5: reopening flushed file at 1732376587237 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4633cbb8: reopening flushed file at 1732376587244 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a720cd3: reopening flushed file at 1732376587248 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37dffbaa: reopening flushed file at 1732376587252 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1747ms, sequenceid=90, compaction requested=false at 1732376587257 (+5 ms)Writing region close event to WAL at 1732376587258 (+1 ms)Running coprocessor post-close hooks at 1732376587261 (+3 ms)Closed at 1732376587261 2024-11-23T15:43:07,262 DEBUG [RS_CLOSE_META-regionserver/b712f9af2c12:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T15:43:07,313 INFO [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(976): stopping server b712f9af2c12,46589,1732376409194; all regions closed. 2024-11-23T15:43:07,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741834_1010 (size=25344) 2024-11-23T15:43:07,324 DEBUG [RS:0;b712f9af2c12:46589 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/oldWALs 2024-11-23T15:43:07,324 INFO [RS:0;b712f9af2c12:46589 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b712f9af2c12%2C46589%2C1732376409194.meta:.meta(num 1732376413364) 2024-11-23T15:43:07,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741832_1008 (size=14123071) 2024-11-23T15:43:07,328 DEBUG [RS:0;b712f9af2c12:46589 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/oldWALs 2024-11-23T15:43:07,328 INFO [RS:0;b712f9af2c12:46589 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b712f9af2c12%2C46589%2C1732376409194:(num 1732376412313) 2024-11-23T15:43:07,328 DEBUG [RS:0;b712f9af2c12:46589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T15:43:07,328 INFO [RS:0;b712f9af2c12:46589 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T15:43:07,328 INFO [RS:0;b712f9af2c12:46589 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T15:43:07,328 INFO [RS:0;b712f9af2c12:46589 {}] hbase.ChoreService(370): Chore service for: regionserver/b712f9af2c12:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T15:43:07,329 INFO [RS:0;b712f9af2c12:46589 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T15:43:07,329 INFO [regionserver/b712f9af2c12:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T15:43:07,329 INFO [RS:0;b712f9af2c12:46589 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46589 2024-11-23T15:43:07,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46589-0x10169a3c09b0001, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b712f9af2c12,46589,1732376409194 2024-11-23T15:43:07,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T15:43:07,387 INFO [RS:0;b712f9af2c12:46589 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T15:43:07,401 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b712f9af2c12,46589,1732376409194] 2024-11-23T15:43:07,411 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b712f9af2c12,46589,1732376409194 already deleted, retry=false 2024-11-23T15:43:07,412 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b712f9af2c12,46589,1732376409194 expired; onlineServers=0 2024-11-23T15:43:07,412 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'b712f9af2c12,42351,1732376408377' ***** 2024-11-23T15:43:07,412 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T15:43:07,413 INFO [M:0;b712f9af2c12:42351 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T15:43:07,413 INFO [M:0;b712f9af2c12:42351 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T15:43:07,413 DEBUG [M:0;b712f9af2c12:42351 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T15:43:07,413 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T15:43:07,413 DEBUG [M:0;b712f9af2c12:42351 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T15:43:07,413 DEBUG [master/b712f9af2c12:0:becomeActiveMaster-HFileCleaner.small.0-1732376412035 {}] cleaner.HFileCleaner(306): Exit Thread[master/b712f9af2c12:0:becomeActiveMaster-HFileCleaner.small.0-1732376412035,5,FailOnTimeoutGroup] 2024-11-23T15:43:07,414 DEBUG [master/b712f9af2c12:0:becomeActiveMaster-HFileCleaner.large.0-1732376412032 {}] cleaner.HFileCleaner(306): Exit Thread[master/b712f9af2c12:0:becomeActiveMaster-HFileCleaner.large.0-1732376412032,5,FailOnTimeoutGroup] 2024-11-23T15:43:07,414 INFO [M:0;b712f9af2c12:42351 {}] hbase.ChoreService(370): Chore service for: master/b712f9af2c12:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T15:43:07,414 INFO [M:0;b712f9af2c12:42351 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T15:43:07,415 DEBUG [M:0;b712f9af2c12:42351 {}] master.HMaster(1795): Stopping service threads 2024-11-23T15:43:07,415 INFO [M:0;b712f9af2c12:42351 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T15:43:07,415 INFO [M:0;b712f9af2c12:42351 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T15:43:07,415 ERROR [M:0;b712f9af2c12:42351 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:46341 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:46341,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-23T15:43:07,416 INFO [M:0;b712f9af2c12:42351 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T15:43:07,416 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T15:43:07,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T15:43:07,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T15:43:07,426 DEBUG [M:0;b712f9af2c12:42351 {}] zookeeper.ZKUtil(347): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T15:43:07,426 WARN [M:0;b712f9af2c12:42351 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T15:43:07,426 INFO [M:0;b712f9af2c12:42351 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/.lastflushedseqids 2024-11-23T15:43:07,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742442_1618 (size=72) 2024-11-23T15:43:07,451 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T15:43:07,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46589-0x10169a3c09b0001, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T15:43:07,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46589-0x10169a3c09b0001, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T15:43:07,502 INFO [RS:0;b712f9af2c12:46589 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T15:43:07,503 INFO [RS:0;b712f9af2c12:46589 {}] regionserver.HRegionServer(1031): Exiting; stopping=b712f9af2c12,46589,1732376409194; zookeeper connection closed. 2024-11-23T15:43:07,503 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@215f7ed5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@215f7ed5 2024-11-23T15:43:07,504 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-23T15:43:07,835 INFO [M:0;b712f9af2c12:42351 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T15:43:07,836 INFO [M:0;b712f9af2c12:42351 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T15:43:07,836 DEBUG [M:0;b712f9af2c12:42351 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T15:43:07,836 INFO [M:0;b712f9af2c12:42351 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:43:07,836 DEBUG [M:0;b712f9af2c12:42351 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:43:07,836 DEBUG [M:0;b712f9af2c12:42351 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T15:43:07,837 DEBUG [M:0;b712f9af2c12:42351 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:43:07,837 INFO [M:0;b712f9af2c12:42351 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=768.01 KB heapSize=944.42 KB 2024-11-23T15:43:07,859 DEBUG [M:0;b712f9af2c12:42351 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e67b0023cc584f219b37c6d742edd64f is 82, key is hbase:meta,,1/info:regioninfo/1732376413448/Put/seqid=0 2024-11-23T15:43:07,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742443_1619 (size=5672) 2024-11-23T15:43:08,263 INFO [M:0;b712f9af2c12:42351 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2175 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e67b0023cc584f219b37c6d742edd64f 2024-11-23T15:43:08,294 DEBUG [M:0;b712f9af2c12:42351 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6337c332a03b4aff84d1b8b6a0810ad5 is 2275, key is \x00\x00\x00\x00\x00\x00\x00\x97/proc:d/1732376559748/Put/seqid=0 2024-11-23T15:43:08,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742444_1620 (size=45517) 2024-11-23T15:43:08,698 INFO [M:0;b712f9af2c12:42351 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=767.45 KB at sequenceid=2175 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6337c332a03b4aff84d1b8b6a0810ad5 2024-11-23T15:43:08,702 INFO [M:0;b712f9af2c12:42351 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6337c332a03b4aff84d1b8b6a0810ad5 2024-11-23T15:43:08,720 DEBUG [M:0;b712f9af2c12:42351 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3f52f4c5828d48d789a9b69ef5576a26 is 69, key is b712f9af2c12,46589,1732376409194/rs:state/1732376412068/Put/seqid=0 2024-11-23T15:43:08,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073742445_1621 (size=5156) 2024-11-23T15:43:08,768 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T15:43:08,768 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-23T15:43:08,769 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-23T15:43:09,125 INFO [M:0;b712f9af2c12:42351 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2175 (bloomFilter=true), to=hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3f52f4c5828d48d789a9b69ef5576a26 2024-11-23T15:43:09,134 DEBUG [M:0;b712f9af2c12:42351 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e67b0023cc584f219b37c6d742edd64f as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e67b0023cc584f219b37c6d742edd64f 2024-11-23T15:43:09,137 INFO [M:0;b712f9af2c12:42351 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e67b0023cc584f219b37c6d742edd64f, entries=8, sequenceid=2175, filesize=5.5 K 2024-11-23T15:43:09,138 DEBUG [M:0;b712f9af2c12:42351 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6337c332a03b4aff84d1b8b6a0810ad5 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6337c332a03b4aff84d1b8b6a0810ad5 2024-11-23T15:43:09,142 INFO [M:0;b712f9af2c12:42351 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6337c332a03b4aff84d1b8b6a0810ad5 2024-11-23T15:43:09,142 INFO [M:0;b712f9af2c12:42351 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6337c332a03b4aff84d1b8b6a0810ad5, entries=178, sequenceid=2175, filesize=44.5 K 2024-11-23T15:43:09,143 DEBUG [M:0;b712f9af2c12:42351 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3f52f4c5828d48d789a9b69ef5576a26 as hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3f52f4c5828d48d789a9b69ef5576a26 2024-11-23T15:43:09,147 INFO [M:0;b712f9af2c12:42351 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46341/user/jenkins/test-data/8c4de2c5-76c7-b096-4d55-36679d30f95e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3f52f4c5828d48d789a9b69ef5576a26, entries=1, sequenceid=2175, filesize=5.0 K 2024-11-23T15:43:09,148 INFO [M:0;b712f9af2c12:42351 {}] regionserver.HRegion(3140): Finished flush of dataSize ~768.01 KB/786438, heapSize ~944.13 KB/966784, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1311ms, sequenceid=2175, compaction requested=false 2024-11-23T15:43:09,149 INFO [M:0;b712f9af2c12:42351 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T15:43:09,149 DEBUG [M:0;b712f9af2c12:42351 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732376587836Disabling compacts and flushes for region at 1732376587836Disabling writes for close at 1732376587836Obtaining lock to block concurrent updates at 1732376587837 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732376587837Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=786438, getHeapSize=967024, getOffHeapSize=0, getCellsCount=2378 at 1732376587838 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732376587839 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732376587839Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732376587858 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732376587858Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732376588271 (+413 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732376588293 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732376588293Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732376588702 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732376588720 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732376588720Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60b7a8e7: reopening flushed file at 1732376589133 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5262b171: reopening flushed file at 1732376589138 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34c7b68f: reopening flushed file at 1732376589142 (+4 ms)Finished flush of dataSize ~768.01 KB/786438, heapSize ~944.13 KB/966784, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1311ms, sequenceid=2175, compaction requested=false at 1732376589148 (+6 ms)Writing region close event to WAL at 1732376589149 (+1 ms)Closed at 1732376589149 2024-11-23T15:43:09,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35405 is added to blk_1073741830_1006 (size=929044) 2024-11-23T15:43:09,152 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T15:43:09,152 INFO [M:0;b712f9af2c12:42351 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T15:43:09,152 INFO [M:0;b712f9af2c12:42351 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42351 2024-11-23T15:43:09,152 INFO [M:0;b712f9af2c12:42351 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T15:43:09,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T15:43:09,303 INFO [M:0;b712f9af2c12:42351 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T15:43:09,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42351-0x10169a3c09b0000, quorum=127.0.0.1:58385, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T15:43:09,312 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T15:43:09,315 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T15:43:09,315 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T15:43:09,316 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T15:43:09,316 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/hadoop.log.dir/,STOPPED} 2024-11-23T15:43:09,319 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T15:43:09,319 WARN [BP-1051640608-172.17.0.2-1732376404362 heartbeating to localhost/127.0.0.1:46341 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T15:43:09,319 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T15:43:09,319 WARN [BP-1051640608-172.17.0.2-1732376404362 heartbeating to localhost/127.0.0.1:46341 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1051640608-172.17.0.2-1732376404362 (Datanode Uuid 637a375a-7b58-4664-9b52-e85f966970ab) service to localhost/127.0.0.1:46341 2024-11-23T15:43:09,323 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/cluster_bcb5f073-d146-1328-644a-da8d77accbc4/data/data1/current/BP-1051640608-172.17.0.2-1732376404362 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T15:43:09,324 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/cluster_bcb5f073-d146-1328-644a-da8d77accbc4/data/data2/current/BP-1051640608-172.17.0.2-1732376404362 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T15:43:09,324 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T15:43:09,331 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T15:43:09,332 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T15:43:09,332 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T15:43:09,332 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T15:43:09,332 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4dca8cd0-a20c-1682-cdbe-46809625921d/hadoop.log.dir/,STOPPED} 2024-11-23T15:43:09,346 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T15:43:09,461 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down